2023-09-06 20:35:20,102 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5b7a5baa 2023-09-06 20:35:20,114 main DEBUG Took 0.010299 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2023-09-06 20:35:20,115 main DEBUG PluginManager 'Core' found 129 plugins 2023-09-06 20:35:20,115 main DEBUG PluginManager 'Level' found 0 plugins 2023-09-06 20:35:20,116 main DEBUG PluginManager 'Lookup' found 16 plugins 2023-09-06 20:35:20,117 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:35:20,126 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2023-09-06 20:35:20,138 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:35:20,140 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:35:20,140 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:35:20,140 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:35:20,141 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:35:20,141 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:35:20,142 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:35:20,142 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:35:20,142 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:35:20,142 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:35:20,143 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:35:20,143 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:35:20,144 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:35:20,144 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:35:20,144 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:35:20,144 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:35:20,145 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:35:20,145 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:35:20,145 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:35:20,146 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:35:20,146 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:35:20,146 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:35:20,147 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:35:20,147 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:35:20,147 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:35:20,147 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2023-09-06 20:35:20,149 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:35:20,151 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2023-09-06 20:35:20,152 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2023-09-06 20:35:20,153 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2023-09-06 20:35:20,154 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2023-09-06 20:35:20,154 main DEBUG PluginManager 'Converter' found 47 plugins 2023-09-06 20:35:20,166 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2023-09-06 20:35:20,169 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2023-09-06 20:35:20,173 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2023-09-06 20:35:20,173 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2023-09-06 20:35:20,173 main DEBUG createAppenders(={Console}) 2023-09-06 20:35:20,174 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5b7a5baa initialized 2023-09-06 20:35:20,174 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5b7a5baa 2023-09-06 20:35:20,175 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5b7a5baa OK. 2023-09-06 20:35:20,175 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2023-09-06 20:35:20,175 main DEBUG OutputStream closed 2023-09-06 20:35:20,176 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2023-09-06 20:35:20,176 main DEBUG Appender DefaultConsole-1 stopped with status true 2023-09-06 20:35:20,176 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@72f926e6 OK 2023-09-06 20:35:20,224 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586 2023-09-06 20:35:20,226 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=StatusLogger 2023-09-06 20:35:20,227 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=ContextSelector 2023-09-06 20:35:20,229 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name= 2023-09-06 20:35:20,229 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.directory 2023-09-06 20:35:20,229 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2023-09-06 20:35:20,230 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.zookeeper 2023-09-06 20:35:20,230 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2023-09-06 20:35:20,230 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2023-09-06 20:35:20,231 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2023-09-06 20:35:20,231 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase 2023-09-06 20:35:20,231 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop 2023-09-06 20:35:20,231 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2023-09-06 20:35:20,232 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2023-09-06 20:35:20,232 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2023-09-06 20:35:20,232 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2023-09-06 20:35:20,232 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2023-09-06 20:35:20,233 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Appenders,name=Console 2023-09-06 20:35:20,236 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-09-06 20:35:20,236 main DEBUG Reconfiguration complete for context[name=1b6d3586] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.6.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@2dde1bff) with optional ClassLoader: null 2023-09-06 20:35:20,237 main DEBUG Shutdown hook enabled. Registering a new one. 2023-09-06 20:35:20,237 main DEBUG LoggerContext[name=1b6d3586, org.apache.logging.log4j.core.LoggerContext@2dde1bff] started OK. 2023-09-06T20:35:20,267 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.backup.TestBackupDelete timeout: 13 mins 2023-09-06 20:35:20,271 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2023-09-06 20:35:20,271 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-09-06T20:35:20,599 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e 2023-09-06T20:35:20,603 DEBUG [Time-limited test {}] impl.BackupManager(127): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner. Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager.Added master procedure manager: org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-09-06T20:35:20,606 DEBUG [Time-limited test {}] impl.BackupManager(157): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager. Added region observer: org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:35:20,607 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2023-09-06T20:35:20,610 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/cluster_4bbcbad3-1dd9-986e-e95b-c3cf1f3afe53, deleteOnExit=true 2023-09-06T20:35:20,610 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2023-09-06T20:35:20,611 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/test.cache.data in system properties and HBase conf 2023-09-06T20:35:20,611 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/hadoop.tmp.dir in system properties and HBase conf 2023-09-06T20:35:20,612 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/hadoop.log.dir in system properties and HBase conf 2023-09-06T20:35:20,613 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/mapreduce.cluster.local.dir in system properties and HBase conf 2023-09-06T20:35:20,613 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-09-06T20:35:20,613 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-09-06T20:35:20,710 WARN [Time-limited test {}] util.NativeCodeLoader(62): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2023-09-06T20:35:21,161 DEBUG [Time-limited test {}] fs.HFileSystem(308): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-09-06T20:35:21,165 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-09-06T20:35:21,166 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-09-06T20:35:21,166 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-09-06T20:35:21,166 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-09-06T20:35:21,167 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-09-06T20:35:21,167 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-09-06T20:35:21,167 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-09-06T20:35:21,167 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/dfs.journalnode.edits.dir in system properties and HBase conf 2023-09-06T20:35:21,168 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-09-06T20:35:21,168 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/nfs.dump.dir in system properties and HBase conf 2023-09-06T20:35:21,168 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/java.io.tmpdir in system properties and HBase conf 2023-09-06T20:35:21,169 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/dfs.journalnode.edits.dir in system properties and HBase conf 2023-09-06T20:35:21,169 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-09-06T20:35:21,169 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2023-09-06T20:35:21,610 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-09-06T20:35:21,614 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-09-06T20:35:21,896 WARN [Time-limited test {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2023-09-06T20:35:22,059 INFO [Time-limited test {}] log.Slf4jLog(67): Logging to org.apache.logging.slf4j.Log4jLogger@95ddad0 via org.mortbay.log.Slf4jLog 2023-09-06T20:35:22,074 WARN [Time-limited test {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-09-06T20:35:22,110 INFO [Time-limited test {}] log.Slf4jLog(67): jetty-6.1.26 2023-09-06T20:35:22,151 INFO [Time-limited test {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/java.io.tmpdir/Jetty_localhost_40509_hdfs____.hws02c/webapp 2023-09-06T20:35:22,293 INFO [Time-limited test {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:40509 2023-09-06T20:35:22,313 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-09-06T20:35:22,313 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-09-06T20:35:22,820 WARN [Listener at localhost/42649 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-09-06T20:35:22,883 WARN [Listener at localhost/42649 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-09-06T20:35:22,900 WARN [Listener at localhost/42649 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-09-06T20:35:22,906 INFO [Listener at localhost/42649 {}] log.Slf4jLog(67): jetty-6.1.26 2023-09-06T20:35:22,912 INFO [Listener at localhost/42649 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/java.io.tmpdir/Jetty_localhost_37389_datanode____.x2rcsp/webapp 2023-09-06T20:35:23,008 INFO [Listener at localhost/42649 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:37389 2023-09-06T20:35:23,296 WARN [Listener at localhost/32815 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-09-06T20:35:23,570 WARN [Thread-52 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-09-06T20:35:23,755 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xb15e34988633d6f8: Processing first storage report for DS-c9c70a6a-ab73-4861-a2f6-0e02c09a2252 from datanode 687aa97e-2a67-4d56-b087-be677eb3d3a5 2023-09-06T20:35:23,757 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xb15e34988633d6f8: from storage DS-c9c70a6a-ab73-4861-a2f6-0e02c09a2252 node DatanodeRegistration(127.0.0.1:36635, datanodeUuid=687aa97e-2a67-4d56-b087-be677eb3d3a5, infoPort=43347, infoSecurePort=0, ipcPort=32815, storageInfo=lv=-57;cid=testClusterID;nsid=1914623259;c=1694032521686), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2023-09-06T20:35:23,757 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xb15e34988633d6f8: Processing first storage report for DS-d06a4857-c2c8-4193-86ae-4f799d4774f0 from datanode 687aa97e-2a67-4d56-b087-be677eb3d3a5 2023-09-06T20:35:23,757 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xb15e34988633d6f8: from storage DS-d06a4857-c2c8-4193-86ae-4f799d4774f0 node DatanodeRegistration(127.0.0.1:36635, datanodeUuid=687aa97e-2a67-4d56-b087-be677eb3d3a5, infoPort=43347, infoSecurePort=0, ipcPort=32815, storageInfo=lv=-57;cid=testClusterID;nsid=1914623259;c=1694032521686), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-09-06T20:35:23,846 DEBUG [Listener at localhost/32815 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e 2023-09-06T20:35:23,925 INFO [Listener at localhost/32815 {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/cluster_4bbcbad3-1dd9-986e-e95b-c3cf1f3afe53/zookeeper_0, clientPort=58778, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/cluster_4bbcbad3-1dd9-986e-e95b-c3cf1f3afe53/zookeeper_0/version-2, dataDirSize=424 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/cluster_4bbcbad3-1dd9-986e-e95b-c3cf1f3afe53/zookeeper_0/version-2, dataLogSize=424 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, serverId=0 2023-09-06T20:35:23,942 INFO [Listener at localhost/32815 {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=58778 2023-09-06T20:35:23,949 INFO [Listener at localhost/32815 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:35:23,952 INFO [Listener at localhost/32815 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:35:24,613 INFO [Listener at localhost/32815 {}] util.FSUtils(461): Created version file at hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0 with version=8 2023-09-06T20:35:24,613 INFO [Listener at localhost/32815 {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/hbase-staging 2023-09-06T20:35:24,630 DEBUG [Listener at localhost/32815 {}] hbase.LocalHBaseCluster(134): Setting Master Port to random. 2023-09-06T20:35:24,630 DEBUG [Listener at localhost/32815 {}] hbase.LocalHBaseCluster(141): Setting RegionServer Port to random. 2023-09-06T20:35:24,630 DEBUG [Listener at localhost/32815 {}] hbase.LocalHBaseCluster(151): Setting RS InfoServer Port to random. 2023-09-06T20:35:24,630 DEBUG [Listener at localhost/32815 {}] hbase.LocalHBaseCluster(159): Setting Master InfoServer Port to random. 2023-09-06T20:35:24,847 DEBUG [Listener at localhost/32815 {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2023-09-06T20:35:25,633 INFO [Listener at localhost/32815 {}] client.ConnectionUtils(128): master/jenkins-hbase3:0 server-side Connection retries=45 2023-09-06T20:35:25,680 INFO [Listener at localhost/32815 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-09-06T20:35:25,680 INFO [Listener at localhost/32815 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-09-06T20:35:25,680 INFO [Listener at localhost/32815 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-09-06T20:35:25,681 INFO [Listener at localhost/32815 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-09-06T20:35:25,681 INFO [Listener at localhost/32815 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-09-06T20:35:25,835 INFO [Listener at localhost/32815 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-09-06T20:35:25,972 INFO [Listener at localhost/32815 {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2023-09-06T20:35:25,990 DEBUG [Listener at localhost/32815 {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2023-09-06T20:35:25,996 INFO [Listener at localhost/32815 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-09-06T20:35:26,047 DEBUG [Listener at localhost/32815 {}] channel.DefaultChannelId(79): -Dio.netty.processId: 5347 (auto-detected) 2023-09-06T20:35:26,048 DEBUG [Listener at localhost/32815 {}] channel.DefaultChannelId(101): -Dio.netty.machineId: 02:42:1c:ff:fe:e2:13:ea (auto-detected) 2023-09-06T20:35:26,085 INFO [Listener at localhost/32815 {}] ipc.NettyRpcServer(192): Bind to /172.31.12.81:42111 2023-09-06T20:35:26,099 INFO [Listener at localhost/32815 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:35:26,101 INFO [Listener at localhost/32815 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:35:26,122 INFO [Listener at localhost/32815 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=master:42111 connecting to ZooKeeper ensemble=127.0.0.1:58778 2023-09-06T20:35:26,171 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): master:421110x0, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-09-06T20:35:26,175 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): master:42111-0x100eefe52de0000 connected 2023-09-06T20:35:26,202 DEBUG [Listener at localhost/32815 {}] zookeeper.ZKUtil(113): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-09-06T20:35:26,206 DEBUG [Listener at localhost/32815 {}] zookeeper.ZKUtil(113): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-09-06T20:35:26,209 DEBUG [Listener at localhost/32815 {}] zookeeper.ZKUtil(113): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-09-06T20:35:26,219 DEBUG [Listener at localhost/32815 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42111 2023-09-06T20:35:26,220 DEBUG [Listener at localhost/32815 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42111 2023-09-06T20:35:26,221 DEBUG [Listener at localhost/32815 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42111 2023-09-06T20:35:26,221 DEBUG [Listener at localhost/32815 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42111 2023-09-06T20:35:26,222 DEBUG [Listener at localhost/32815 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42111 2023-09-06T20:35:26,275 INFO [Listener at localhost/32815 {}] log.Log(170): Logging initialized @6873ms to org.apache.hbase.thirdparty.org.eclipse.jetty.util.log.Slf4jLog 2023-09-06T20:35:26,395 INFO [Listener at localhost/32815 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-09-06T20:35:26,396 INFO [Listener at localhost/32815 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-09-06T20:35:26,397 INFO [Listener at localhost/32815 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-09-06T20:35:26,399 INFO [Listener at localhost/32815 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2023-09-06T20:35:26,399 INFO [Listener at localhost/32815 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-09-06T20:35:26,399 INFO [Listener at localhost/32815 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-09-06T20:35:26,402 INFO [Listener at localhost/32815 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-09-06T20:35:26,551 INFO [Listener at localhost/32815 {}] http.HttpServer(1219): Jetty bound to port 42809 2023-09-06T20:35:26,553 INFO [Listener at localhost/32815 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-09-06T20:35:26,586 INFO [Listener at localhost/32815 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-09-06T20:35:26,590 INFO [Listener at localhost/32815 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@1a69b8b4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/hadoop.log.dir/,AVAILABLE} 2023-09-06T20:35:26,591 INFO [Listener at localhost/32815 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-09-06T20:35:26,591 INFO [Listener at localhost/32815 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@30251af4{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-09-06T20:35:26,665 INFO [Listener at localhost/32815 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-09-06T20:35:26,680 INFO [Listener at localhost/32815 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-09-06T20:35:26,680 INFO [Listener at localhost/32815 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-09-06T20:35:26,682 INFO [Listener at localhost/32815 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2023-09-06T20:35:26,690 INFO [Listener at localhost/32815 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-09-06T20:35:27,818 INFO [Listener at localhost/32815 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@3ad8d793{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-09-06T20:35:27,831 INFO [Listener at localhost/32815 {}] server.AbstractConnector(333): Started ServerConnector@164c99cb{HTTP/1.1, (http/1.1)}{0.0.0.0:42809} 2023-09-06T20:35:27,832 INFO [Listener at localhost/32815 {}] server.Server(415): Started @8429ms 2023-09-06T20:35:27,836 INFO [Listener at localhost/32815 {}] master.HMaster(485): hbase.rootdir=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0, hbase.cluster.distributed=false 2023-09-06T20:35:27,899 INFO [Listener at localhost/32815 {}] client.ConnectionUtils(128): regionserver/jenkins-hbase3:0 server-side Connection retries=45 2023-09-06T20:35:27,899 INFO [Listener at localhost/32815 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-09-06T20:35:27,900 INFO [Listener at localhost/32815 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-09-06T20:35:27,900 INFO [Listener at localhost/32815 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-09-06T20:35:27,900 INFO [Listener at localhost/32815 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-09-06T20:35:27,900 INFO [Listener at localhost/32815 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-09-06T20:35:27,908 INFO [Listener at localhost/32815 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-09-06T20:35:27,910 INFO [Listener at localhost/32815 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-09-06T20:35:27,911 INFO [Listener at localhost/32815 {}] ipc.NettyRpcServer(192): Bind to /172.31.12.81:46803 2023-09-06T20:35:27,913 INFO [Listener at localhost/32815 {}] hfile.BlockCacheFactory(127): Allocating BlockCache size=782.40 MB, blockSize=64 KB 2023-09-06T20:35:27,920 DEBUG [Listener at localhost/32815 {}] mob.MobFileCache(120): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2023-09-06T20:35:27,921 INFO [Listener at localhost/32815 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:35:27,924 INFO [Listener at localhost/32815 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:35:27,926 INFO [Listener at localhost/32815 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=regionserver:46803 connecting to ZooKeeper ensemble=127.0.0.1:58778 2023-09-06T20:35:27,929 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:468030x0, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-09-06T20:35:27,930 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): regionserver:46803-0x100eefe52de0001 connected 2023-09-06T20:35:27,930 DEBUG [Listener at localhost/32815 {}] zookeeper.ZKUtil(113): regionserver:46803-0x100eefe52de0001, quorum=127.0.0.1:58778, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-09-06T20:35:27,932 DEBUG [Listener at localhost/32815 {}] zookeeper.ZKUtil(113): regionserver:46803-0x100eefe52de0001, quorum=127.0.0.1:58778, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-09-06T20:35:27,933 DEBUG [Listener at localhost/32815 {}] zookeeper.ZKUtil(113): regionserver:46803-0x100eefe52de0001, quorum=127.0.0.1:58778, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-09-06T20:35:27,933 DEBUG [Listener at localhost/32815 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46803 2023-09-06T20:35:27,933 DEBUG [Listener at localhost/32815 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46803 2023-09-06T20:35:27,934 DEBUG [Listener at localhost/32815 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46803 2023-09-06T20:35:27,934 DEBUG [Listener at localhost/32815 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46803 2023-09-06T20:35:27,935 DEBUG [Listener at localhost/32815 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46803 2023-09-06T20:35:27,938 INFO [Listener at localhost/32815 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-09-06T20:35:27,938 INFO [Listener at localhost/32815 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-09-06T20:35:27,938 INFO [Listener at localhost/32815 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-09-06T20:35:27,939 INFO [Listener at localhost/32815 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2023-09-06T20:35:27,939 INFO [Listener at localhost/32815 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-09-06T20:35:27,939 INFO [Listener at localhost/32815 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-09-06T20:35:27,940 INFO [Listener at localhost/32815 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-09-06T20:35:27,942 INFO [Listener at localhost/32815 {}] http.HttpServer(1219): Jetty bound to port 33351 2023-09-06T20:35:27,942 INFO [Listener at localhost/32815 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-09-06T20:35:27,943 INFO [Listener at localhost/32815 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-09-06T20:35:27,943 INFO [Listener at localhost/32815 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@17ea525b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/hadoop.log.dir/,AVAILABLE} 2023-09-06T20:35:27,944 INFO [Listener at localhost/32815 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-09-06T20:35:27,944 INFO [Listener at localhost/32815 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@1a457c7a{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-09-06T20:35:27,956 INFO [Listener at localhost/32815 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-09-06T20:35:27,957 INFO [Listener at localhost/32815 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-09-06T20:35:27,958 INFO [Listener at localhost/32815 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-09-06T20:35:27,958 INFO [Listener at localhost/32815 {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2023-09-06T20:35:27,959 INFO [Listener at localhost/32815 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-09-06T20:35:27,963 INFO [Listener at localhost/32815 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@dfee667{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-09-06T20:35:27,964 INFO [Listener at localhost/32815 {}] server.AbstractConnector(333): Started ServerConnector@3271f28d{HTTP/1.1, (http/1.1)}{0.0.0.0:33351} 2023-09-06T20:35:27,964 INFO [Listener at localhost/32815 {}] server.Server(415): Started @8562ms 2023-09-06T20:35:27,970 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-09-06T20:35:27,973 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@585959c5{HTTP/1.1, (http/1.1)}{0.0.0.0:38611} 2023-09-06T20:35:27,973 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(415): Started @8571ms 2023-09-06T20:35:27,973 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2388): Adding backup master ZNode /1/backup-masters/jenkins-hbase3.apache.org,42111,1694032524831 2023-09-06T20:35:27,988 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:46803-0x100eefe52de0001, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-09-06T20:35:27,988 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-09-06T20:35:27,990 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Set watcher on existing znode=/1/backup-masters/jenkins-hbase3.apache.org,42111,1694032524831 2023-09-06T20:35:27,991 DEBUG [M:0;jenkins-hbase3:42111 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;jenkins-hbase3:42111 2023-09-06T20:35:28,008 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:46803-0x100eefe52de0001, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-09-06T20:35:28,008 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-09-06T20:35:28,008 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:46803-0x100eefe52de0001, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:35:28,008 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:35:28,009 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Set watcher on existing znode=/1/master 2023-09-06T20:35:28,011 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /1/backup-masters/jenkins-hbase3.apache.org,42111,1694032524831 from backup master directory 2023-09-06T20:35:28,011 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Set watcher on existing znode=/1/master 2023-09-06T20:35:28,015 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/backup-masters/jenkins-hbase3.apache.org,42111,1694032524831 2023-09-06T20:35:28,015 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:46803-0x100eefe52de0001, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-09-06T20:35:28,016 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-09-06T20:35:28,016 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-09-06T20:35:28,016 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=jenkins-hbase3.apache.org,42111,1694032524831 2023-09-06T20:35:28,020 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 352, initial count 0 2023-09-06T20:35:28,021 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 391, initial count 0 2023-09-06T20:35:28,500 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] util.FSUtils(610): Created cluster ID file at hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/hbase.id with ID: 7bbce971-8494-4dfe-b095-7ab4072d8230 2023-09-06T20:35:28,542 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:35:28,556 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:46803-0x100eefe52de0001, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:35:28,556 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:35:28,599 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(355): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:35:28,601 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2023-09-06T20:35:28,622 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(270): ClientProtocol::create wrong number of arguments, should be hadoop 3.2 or below 2023-09-06T20:35:28,622 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(276): ClientProtocol::create wrong number of arguments, should be hadoop 2.x 2023-09-06T20:35:28,624 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(285): can not find SHOULD_REPLICATE flag, should be hadoop 2.x java.lang.IllegalArgumentException: No enum constant org.apache.hadoop.fs.CreateFlag.SHOULD_REPLICATE at java.lang.Enum.valueOf(Enum.java:238) ~[?:1.8.0_362] at org.apache.hadoop.fs.CreateFlag.valueOf(CreateFlag.java:63) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.loadShouldReplicateFlag(FanOutOneBlockAsyncDFSOutputHelper.java:283) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.(FanOutOneBlockAsyncDFSOutputHelper.java:310) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:140) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:951) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2413) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:574) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:571) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:35:28,636 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:141) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:951) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2413) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:574) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:571) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:35:28,638 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-09-06T20:35:29,111 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7093): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/MasterData/data/master/store 2023-09-06T20:35:29,136 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2023-09-06T20:35:29,136 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:35:29,137 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-09-06T20:35:29,137 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:35:29,137 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:35:29,137 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-09-06T20:35:29,137 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:35:29,138 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:35:29,138 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-09-06T20:35:29,140 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(232): failed to clean up initializing flag: hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/MasterData/data/master/store/.initializing 2023-09-06T20:35:29,140 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(202): WALDir=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/MasterData/WALs/jenkins-hbase3.apache.org,42111,1694032524831 2023-09-06T20:35:29,150 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-09-06T20:35:29,164 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C42111%2C1694032524831, suffix=, logDir=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/MasterData/WALs/jenkins-hbase3.apache.org,42111,1694032524831, archiveDir=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/MasterData/oldWALs, maxLogs=10 2023-09-06T20:35:29,202 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/MasterData/WALs/jenkins-hbase3.apache.org,42111,1694032524831/jenkins-hbase3.apache.org%2C42111%2C1694032524831.1694032529171, exclude list is [], retry=0 2023-09-06T20:35:29,233 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36635,DS-c9c70a6a-ab73-4861-a2f6-0e02c09a2252,DISK] 2023-09-06T20:35:29,241 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(123): Hadoop 3.2 and below use unshaded protobuf. java.lang.ClassNotFoundException: org.apache.hadoop.thirdparty.protobuf.MessageLite at java.net.URLClassLoader.findClass(URLClassLoader.java:387) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:418) ~[?:1.8.0_362] at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ~[?:1.8.0_362] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.ProtobufDecoder.(ProtobufDecoder.java:118) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.processWriteBlockResponse(FanOutOneBlockAsyncDFSOutputHelper.java:346) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$100(FanOutOneBlockAsyncDFSOutputHelper.java:120) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$4.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:430) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:185) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:35) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hadoop.hbase.util.NettyFutureUtils.addListener(NettyFutureUtils.java:52) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:424) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$300(FanOutOneBlockAsyncDFSOutputHelper.java:120) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:482) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:477) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:653) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:691) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:489) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:397) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.4.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:35:29,316 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/MasterData/WALs/jenkins-hbase3.apache.org,42111,1694032524831/jenkins-hbase3.apache.org%2C42111%2C1694032524831.1694032529171 2023-09-06T20:35:29,317 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36635,DS-c9c70a6a-ab73-4861-a2f6-0e02c09a2252,DISK]] 2023-09-06T20:35:29,318 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7254): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:35:29,318 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:35:29,322 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7296): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:35:29,324 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7299): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:35:29,392 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:35:29,428 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2023-09-06T20:35:29,433 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:29,437 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-06T20:35:29,437 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:35:29,440 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2023-09-06T20:35:29,440 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:29,441 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:35:29,442 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:35:29,444 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2023-09-06T20:35:29,444 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:29,445 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:35:29,445 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:35:29,447 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2023-09-06T20:35:29,447 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:29,448 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:35:29,453 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:35:29,454 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:35:29,472 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2023-09-06T20:35:29,476 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1076): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:35:29,480 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:35:29,482 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1093): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9692887200, jitterRate=-0.09727953374385834}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2023-09-06T20:35:29,489 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(992): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2023-09-06T20:35:29,490 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2023-09-06T20:35:29,522 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bbc118f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:35:29,580 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(860): No meta location available on zookeeper, skip migrating... 2023-09-06T20:35:29,602 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2023-09-06T20:35:29,602 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(563): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2023-09-06T20:35:29,605 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2023-09-06T20:35:29,607 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(583): Recovered RegionProcedureStore lease in 1 msec 2023-09-06T20:35:29,613 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(597): Loaded RegionProcedureStore in 6 msec 2023-09-06T20:35:29,613 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2023-09-06T20:35:29,648 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2023-09-06T20:35:29,665 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Unable to get data of znode /1/balancer because node does not exist (not necessarily an error) 2023-09-06T20:35:29,667 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/balancer already deleted, retry=false 2023-09-06T20:35:29,670 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(150): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2023-09-06T20:35:29,671 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Unable to get data of znode /1/normalizer because node does not exist (not necessarily an error) 2023-09-06T20:35:29,672 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/normalizer already deleted, retry=false 2023-09-06T20:35:29,675 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2023-09-06T20:35:29,682 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Unable to get data of znode /1/switch/split because node does not exist (not necessarily an error) 2023-09-06T20:35:29,683 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/split already deleted, retry=false 2023-09-06T20:35:29,684 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Unable to get data of znode /1/switch/merge because node does not exist (not necessarily an error) 2023-09-06T20:35:29,687 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/merge already deleted, retry=false 2023-09-06T20:35:29,703 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Unable to get data of znode /1/snapshot-cleanup because node does not exist (not necessarily an error) 2023-09-06T20:35:29,704 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/snapshot-cleanup already deleted, retry=false 2023-09-06T20:35:29,710 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:46803-0x100eefe52de0001, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-09-06T20:35:29,710 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-09-06T20:35:29,710 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:46803-0x100eefe52de0001, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:35:29,710 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:35:29,710 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(804): Active/primary master=jenkins-hbase3.apache.org,42111,1694032524831, sessionid=0x100eefe52de0000, setting cluster-up flag (Was=false) 2023-09-06T20:35:29,720 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2023-09-06T20:35:29,728 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:46803-0x100eefe52de0001, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:35:29,728 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:35:29,733 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/flush-table-proc/acquired, /1/flush-table-proc/reached, /1/flush-table-proc/abort 2023-09-06T20:35:29,734 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,42111,1694032524831 2023-09-06T20:35:29,738 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:35:29,738 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:46803-0x100eefe52de0001, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:35:29,748 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/rolllog-proc/acquired, /1/rolllog-proc/reached, /1/rolllog-proc/abort 2023-09-06T20:35:29,750 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,42111,1694032524831 2023-09-06T20:35:29,755 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:35:29,755 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:46803-0x100eefe52de0001, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:35:29,760 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/online-snapshot/acquired, /1/online-snapshot/reached, /1/online-snapshot/abort 2023-09-06T20:35:29,762 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,42111,1694032524831 2023-09-06T20:35:29,878 DEBUG [RS:0;jenkins-hbase3:46803 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;jenkins-hbase3:46803 2023-09-06T20:35:29,880 INFO [RS:0;jenkins-hbase3:46803 {}] regionserver.HRegionServer(997): ClusterId : 7bbce971-8494-4dfe-b095-7ab4072d8230 2023-09-06T20:35:29,882 INFO [RS:0;jenkins-hbase3:46803 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2023-09-06T20:35:29,884 DEBUG [RS:0;jenkins-hbase3:46803 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2023-09-06T20:35:29,890 DEBUG [RS:0;jenkins-hbase3:46803 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2023-09-06T20:35:29,890 DEBUG [RS:0;jenkins-hbase3:46803 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2023-09-06T20:35:29,894 DEBUG [RS:0;jenkins-hbase3:46803 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2023-09-06T20:35:29,894 DEBUG [RS:0;jenkins-hbase3:46803 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2023-09-06T20:35:29,898 DEBUG [RS:0;jenkins-hbase3:46803 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2023-09-06T20:35:29,899 DEBUG [RS:0;jenkins-hbase3:46803 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1726f565, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:35:29,902 DEBUG [RS:0;jenkins-hbase3:46803 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49d53748, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-09-06T20:35:29,904 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2023-09-06T20:35:29,909 INFO [RS:0;jenkins-hbase3:46803 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2023-09-06T20:35:29,910 INFO [RS:0;jenkins-hbase3:46803 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2023-09-06T20:35:29,910 DEBUG [RS:0;jenkins-hbase3:46803 {}] regionserver.HRegionServer(1079): About to register with Master. 2023-09-06T20:35:29,913 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(574): slop=0.2 2023-09-06T20:35:29,914 INFO [RS:0;jenkins-hbase3:46803 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase3.apache.org,42111,1694032524831 with isa=jenkins-hbase3.apache.org/172.31.12.81:46803, startcode=1694032527898 2023-09-06T20:35:29,919 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(285): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2023-09-06T20:35:29,929 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: jenkins-hbase3.apache.org,42111,1694032524831 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2023-09-06T20:35:29,933 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-09-06T20:35:29,934 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-09-06T20:35:29,934 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-09-06T20:35:29,934 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-09-06T20:35:29,934 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/jenkins-hbase3:0, corePoolSize=10, maxPoolSize=10 2023-09-06T20:35:29,934 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:35:29,934 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-09-06T20:35:29,935 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:35:29,936 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1694032559936 2023-09-06T20:35:29,939 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2023-09-06T20:35:29,943 DEBUG [RS:0;jenkins-hbase3:46803 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-09-06T20:35:29,944 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2023-09-06T20:35:29,945 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2023-09-06T20:35:29,946 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2023-09-06T20:35:29,952 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:29,952 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-09-06T20:35:29,957 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2023-09-06T20:35:29,957 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2023-09-06T20:35:29,958 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2023-09-06T20:35:29,959 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2023-09-06T20:35:29,959 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2023-09-06T20:35:29,960 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-09-06T20:35:29,967 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2023-09-06T20:35:29,973 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2023-09-06T20:35:29,974 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2023-09-06T20:35:29,975 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2023-09-06T20:35:29,975 INFO [PEWorker-1 {}] regionserver.HRegion(7075): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0 2023-09-06T20:35:29,986 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ReadOnlyZKClient(139): Connect 0x7bf9bd9b to 127.0.0.1:58778 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:35:30,037 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a527840, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:35:30,037 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-09-06T20:35:30,046 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2023-09-06T20:35:30,047 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2023-09-06T20:35:30,048 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:47887, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-09-06T20:35:30,054 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1694032530049,5,FailOnTimeoutGroup] 2023-09-06T20:35:30,055 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1694032530055,5,FailOnTimeoutGroup] 2023-09-06T20:35:30,055 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-09-06T20:35:30,055 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1635): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2023-09-06T20:35:30,057 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2023-09-06T20:35:30,058 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2023-09-06T20:35:30,065 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42111 {}] master.ServerManager(403): Registering regionserver=jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:30,096 DEBUG [RS:0;jenkins-hbase3:46803 {}] regionserver.HRegionServer(1705): Config from master: hbase.rootdir=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0 2023-09-06T20:35:30,096 DEBUG [RS:0;jenkins-hbase3:46803 {}] regionserver.HRegionServer(1705): Config from master: fs.defaultFS=hdfs://localhost:42649 2023-09-06T20:35:30,096 DEBUG [RS:0;jenkins-hbase3:46803 {}] regionserver.HRegionServer(1705): Config from master: hbase.master.info.port=42809 2023-09-06T20:35:30,104 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2023-09-06T20:35:30,105 DEBUG [RS:0;jenkins-hbase3:46803 {}] zookeeper.ZKUtil(111): regionserver:46803-0x100eefe52de0001, quorum=127.0.0.1:58778, baseZNode=/1 Set watcher on existing znode=/1/rs/jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:30,105 WARN [RS:0;jenkins-hbase3:46803 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-09-06T20:35:30,105 INFO [RS:0;jenkins-hbase3:46803 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-09-06T20:35:30,106 DEBUG [RS:0;jenkins-hbase3:46803 {}] regionserver.HRegionServer(2060): logDir=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/WALs/jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:30,109 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [jenkins-hbase3.apache.org,46803,1694032527898] 2023-09-06T20:35:30,130 DEBUG [RS:0;jenkins-hbase3:46803 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2023-09-06T20:35:30,152 INFO [RS:0;jenkins-hbase3:46803 {}] regionserver.MetricsRegionServerWrapperImpl(165): Computing regionserver metrics every 5000 milliseconds 2023-09-06T20:35:30,172 INFO [RS:0;jenkins-hbase3:46803 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=782.4 M, globalMemStoreLimitLowMark=743.3 M, Offheap=false 2023-09-06T20:35:30,178 INFO [RS:0;jenkins-hbase3:46803 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2023-09-06T20:35:30,178 INFO [RS:0;jenkins-hbase3:46803 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2023-09-06T20:35:30,179 INFO [RS:0;jenkins-hbase3:46803 {}] regionserver.HRegionServer$CompactionChecker(1947): CompactionChecker runs every PT1S 2023-09-06T20:35:30,191 INFO [RS:0;jenkins-hbase3:46803 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2023-09-06T20:35:30,191 DEBUG [RS:0;jenkins-hbase3:46803 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:35:30,192 DEBUG [RS:0;jenkins-hbase3:46803 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:35:30,192 DEBUG [RS:0;jenkins-hbase3:46803 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:35:30,192 DEBUG [RS:0;jenkins-hbase3:46803 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:35:30,192 DEBUG [RS:0;jenkins-hbase3:46803 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:35:30,192 DEBUG [RS:0;jenkins-hbase3:46803 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-09-06T20:35:30,192 DEBUG [RS:0;jenkins-hbase3:46803 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:35:30,192 DEBUG [RS:0;jenkins-hbase3:46803 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:35:30,193 DEBUG [RS:0;jenkins-hbase3:46803 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:35:30,193 DEBUG [RS:0;jenkins-hbase3:46803 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:35:30,193 DEBUG [RS:0;jenkins-hbase3:46803 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:35:30,193 DEBUG [RS:0;jenkins-hbase3:46803 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-09-06T20:35:30,193 DEBUG [RS:0;jenkins-hbase3:46803 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-09-06T20:35:30,194 INFO [RS:0;jenkins-hbase3:46803 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2023-09-06T20:35:30,194 INFO [RS:0;jenkins-hbase3:46803 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2023-09-06T20:35:30,194 INFO [RS:0;jenkins-hbase3:46803 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2023-09-06T20:35:30,194 INFO [RS:0;jenkins-hbase3:46803 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2023-09-06T20:35:30,194 INFO [RS:0;jenkins-hbase3:46803 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,46803,1694032527898-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-09-06T20:35:30,210 INFO [RS:0;jenkins-hbase3:46803 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2023-09-06T20:35:30,213 INFO [RS:0;jenkins-hbase3:46803 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,46803,1694032527898-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2023-09-06T20:35:30,232 INFO [RS:0;jenkins-hbase3:46803 {}] regionserver.Replication(204): jenkins-hbase3.apache.org,46803,1694032527898 started 2023-09-06T20:35:30,232 INFO [RS:0;jenkins-hbase3:46803 {}] regionserver.HRegionServer(1747): Serving as jenkins-hbase3.apache.org,46803,1694032527898, RpcServer on jenkins-hbase3.apache.org/172.31.12.81:46803, sessionid=0x100eefe52de0001 2023-09-06T20:35:30,233 DEBUG [RS:0;jenkins-hbase3:46803 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2023-09-06T20:35:30,233 DEBUG [RS:0;jenkins-hbase3:46803 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:30,233 DEBUG [RS:0;jenkins-hbase3:46803 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,46803,1694032527898' 2023-09-06T20:35:30,233 DEBUG [RS:0;jenkins-hbase3:46803 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/flush-table-proc/abort' 2023-09-06T20:35:30,234 DEBUG [RS:0;jenkins-hbase3:46803 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/flush-table-proc/acquired' 2023-09-06T20:35:30,235 DEBUG [RS:0;jenkins-hbase3:46803 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2023-09-06T20:35:30,235 DEBUG [RS:0;jenkins-hbase3:46803 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2023-09-06T20:35:30,235 DEBUG [RS:0;jenkins-hbase3:46803 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,46803,1694032527898' 2023-09-06T20:35:30,235 DEBUG [RS:0;jenkins-hbase3:46803 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:35:30,235 DEBUG [RS:0;jenkins-hbase3:46803 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:35:30,236 INFO [RS:0;jenkins-hbase3:46803 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2023-09-06T20:35:30,236 DEBUG [RS:0;jenkins-hbase3:46803 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2023-09-06T20:35:30,236 DEBUG [RS:0;jenkins-hbase3:46803 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2023-09-06T20:35:30,236 DEBUG [RS:0;jenkins-hbase3:46803 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:30,236 DEBUG [RS:0;jenkins-hbase3:46803 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,46803,1694032527898' 2023-09-06T20:35:30,236 DEBUG [RS:0;jenkins-hbase3:46803 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/online-snapshot/abort' 2023-09-06T20:35:30,237 DEBUG [RS:0;jenkins-hbase3:46803 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/online-snapshot/acquired' 2023-09-06T20:35:30,237 DEBUG [RS:0;jenkins-hbase3:46803 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2023-09-06T20:35:30,237 INFO [RS:0;jenkins-hbase3:46803 {}] quotas.RegionServerRpcQuotaManager(63): Quota support disabled 2023-09-06T20:35:30,237 INFO [RS:0;jenkins-hbase3:46803 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2023-09-06T20:35:30,348 INFO [RS:0;jenkins-hbase3:46803 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-09-06T20:35:30,352 INFO [RS:0;jenkins-hbase3:46803 {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C46803%2C1694032527898, suffix=, logDir=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/WALs/jenkins-hbase3.apache.org,46803,1694032527898, archiveDir=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/oldWALs, maxLogs=32 2023-09-06T20:35:30,364 DEBUG [RS:0;jenkins-hbase3:46803 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/WALs/jenkins-hbase3.apache.org,46803,1694032527898/jenkins-hbase3.apache.org%2C46803%2C1694032527898.1694032530354, exclude list is [], retry=0 2023-09-06T20:35:30,370 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36635,DS-c9c70a6a-ab73-4861-a2f6-0e02c09a2252,DISK] 2023-09-06T20:35:30,374 INFO [RS:0;jenkins-hbase3:46803 {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/WALs/jenkins-hbase3.apache.org,46803,1694032527898/jenkins-hbase3.apache.org%2C46803%2C1694032527898.1694032530354 2023-09-06T20:35:30,374 DEBUG [RS:0;jenkins-hbase3:46803 {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36635,DS-c9c70a6a-ab73-4861-a2f6-0e02c09a2252,DISK]] 2023-09-06T20:35:30,427 DEBUG [PEWorker-1 {}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:35:30,430 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-09-06T20:35:30,432 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-09-06T20:35:30,432 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:30,433 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-06T20:35:30,433 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-09-06T20:35:30,435 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-09-06T20:35:30,435 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:30,436 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-06T20:35:30,436 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-09-06T20:35:30,438 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-09-06T20:35:30,438 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:30,439 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-06T20:35:30,440 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/hbase/meta/1588230740 2023-09-06T20:35:30,441 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/hbase/meta/1588230740 2023-09-06T20:35:30,443 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-09-06T20:35:30,445 DEBUG [PEWorker-1 {}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-09-06T20:35:30,449 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:35:30,450 INFO [PEWorker-1 {}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11012178880, jitterRate=0.025589078664779663}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-09-06T20:35:30,452 DEBUG [PEWorker-1 {}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-09-06T20:35:30,452 DEBUG [PEWorker-1 {}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-09-06T20:35:30,452 INFO [PEWorker-1 {}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-09-06T20:35:30,452 DEBUG [PEWorker-1 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-09-06T20:35:30,452 DEBUG [PEWorker-1 {}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-09-06T20:35:30,452 DEBUG [PEWorker-1 {}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-09-06T20:35:30,453 INFO [PEWorker-1 {}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-09-06T20:35:30,453 DEBUG [PEWorker-1 {}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-09-06T20:35:30,457 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2023-09-06T20:35:30,457 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2023-09-06T20:35:30,466 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2023-09-06T20:35:30,478 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2023-09-06T20:35:30,482 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2023-09-06T20:35:30,634 DEBUG [jenkins-hbase3:42111 {}] assignment.AssignmentManager(2303): Processing assignQueue; systemServersCount=1, allServersCount=1 2023-09-06T20:35:30,642 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:30,648 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,46803,1694032527898, state=OPENING 2023-09-06T20:35:30,658 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2023-09-06T20:35:30,666 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:46803-0x100eefe52de0001, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:35:30,666 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:35:30,667 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-09-06T20:35:30,667 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-09-06T20:35:30,670 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,46803,1694032527898}] 2023-09-06T20:35:30,860 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(721): New admin connection to jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:30,863 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=AdminService, sasl=false 2023-09-06T20:35:30,867 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:51432, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2023-09-06T20:35:30,879 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(130): Open hbase:meta,,1.1588230740 2023-09-06T20:35:30,880 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-09-06T20:35:30,881 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2023-09-06T20:35:30,883 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C46803%2C1694032527898.meta, suffix=.meta, logDir=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/WALs/jenkins-hbase3.apache.org,46803,1694032527898, archiveDir=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/oldWALs, maxLogs=32 2023-09-06T20:35:30,896 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/WALs/jenkins-hbase3.apache.org,46803,1694032527898/jenkins-hbase3.apache.org%2C46803%2C1694032527898.meta.1694032530884.meta, exclude list is [], retry=0 2023-09-06T20:35:30,900 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36635,DS-c9c70a6a-ab73-4861-a2f6-0e02c09a2252,DISK] 2023-09-06T20:35:30,904 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/WALs/jenkins-hbase3.apache.org,46803,1694032527898/jenkins-hbase3.apache.org%2C46803%2C1694032527898.meta.1694032530884.meta 2023-09-06T20:35:30,904 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36635,DS-c9c70a6a-ab73-4861-a2f6-0e02c09a2252,DISK]] 2023-09-06T20:35:30,904 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7254): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:35:30,909 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:35:30,910 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2023-09-06T20:35:30,928 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7955): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2023-09-06T20:35:30,928 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(393): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2023-09-06T20:35:30,933 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2023-09-06T20:35:30,933 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:35:30,933 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7296): checking encryption for 1588230740 2023-09-06T20:35:30,933 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7299): checking classloading for 1588230740 2023-09-06T20:35:30,936 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-09-06T20:35:30,937 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-09-06T20:35:30,937 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:30,938 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-06T20:35:30,938 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-09-06T20:35:30,939 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-09-06T20:35:30,939 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:30,940 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-06T20:35:30,940 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-09-06T20:35:30,941 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-09-06T20:35:30,941 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:30,942 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-06T20:35:30,943 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/hbase/meta/1588230740 2023-09-06T20:35:30,972 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/hbase/meta/1588230740 2023-09-06T20:35:30,977 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-09-06T20:35:30,979 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-09-06T20:35:30,981 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10846231680, jitterRate=0.01013404130935669}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-09-06T20:35:30,982 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-09-06T20:35:30,995 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1694032530851 2023-09-06T20:35:31,021 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:meta,,1.1588230740 2023-09-06T20:35:31,023 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(158): Opened hbase:meta,,1.1588230740 2023-09-06T20:35:31,024 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:31,026 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,46803,1694032527898, state=OPEN 2023-09-06T20:35:31,029 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-09-06T20:35:31,029 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:46803-0x100eefe52de0001, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-09-06T20:35:31,029 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-09-06T20:35:31,029 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-09-06T20:35:31,033 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=3, resume processing ppid=2 2023-09-06T20:35:31,034 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,46803,1694032527898 in 359 msec 2023-09-06T20:35:31,043 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=2, resume processing ppid=1 2023-09-06T20:35:31,044 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 569 msec 2023-09-06T20:35:31,049 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 1.2220 sec 2023-09-06T20:35:31,050 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1063): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1694032531049, completionTime=-1 2023-09-06T20:35:31,050 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ServerManager(830): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2023-09-06T20:35:31,050 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1629): Joining cluster... 2023-09-06T20:35:31,261 DEBUG [hconnection-0xc2c31e7-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:35:31,265 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:51438, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:35:31,283 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1641): Number of RegionServers=1 2023-09-06T20:35:31,284 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1694032591283 2023-09-06T20:35:31,284 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1694032651284 2023-09-06T20:35:31,284 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1648): Joined the cluster in 233 msec 2023-09-06T20:35:31,309 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,42111,1694032524831-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2023-09-06T20:35:31,310 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,42111,1694032524831-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2023-09-06T20:35:31,310 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,42111,1694032524831-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2023-09-06T20:35:31,312 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-jenkins-hbase3:42111, period=300000, unit=MILLISECONDS is enabled. 2023-09-06T20:35:31,312 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2023-09-06T20:35:31,319 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] janitor.CatalogJanitor(175): 2023-09-06T20:35:31,332 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2023-09-06T20:35:31,334 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2368): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-09-06T20:35:31,343 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2023-09-06T20:35:31,347 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:35:31,349 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:31,350 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:35:31,770 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 0489809eda53d4f420cc714a3ab0c1bc, NAME => 'hbase:namespace,,1694032531333.0489809eda53d4f420cc714a3ab0c1bc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0 2023-09-06T20:35:32,182 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:namespace,,1694032531333.0489809eda53d4f420cc714a3ab0c1bc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:35:32,182 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1668): Closing 0489809eda53d4f420cc714a3ab0c1bc, disabling compactions & flushes 2023-09-06T20:35:32,182 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:namespace,,1694032531333.0489809eda53d4f420cc714a3ab0c1bc. 2023-09-06T20:35:32,182 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1694032531333.0489809eda53d4f420cc714a3ab0c1bc. 2023-09-06T20:35:32,183 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1694032531333.0489809eda53d4f420cc714a3ab0c1bc. after waiting 0 ms 2023-09-06T20:35:32,183 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1694032531333.0489809eda53d4f420cc714a3ab0c1bc. 2023-09-06T20:35:32,183 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1909): Closed hbase:namespace,,1694032531333.0489809eda53d4f420cc714a3ab0c1bc. 2023-09-06T20:35:32,183 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1622): Region close journal for 0489809eda53d4f420cc714a3ab0c1bc: 2023-09-06T20:35:32,186 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:35:32,201 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1694032531333.0489809eda53d4f420cc714a3ab0c1bc.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1694032532188"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032532188"}]},"ts":"1694032532188"} 2023-09-06T20:35:32,242 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:35:32,244 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:35:32,251 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032532244"}]},"ts":"1694032532244"} 2023-09-06T20:35:32,256 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2023-09-06T20:35:32,262 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=0489809eda53d4f420cc714a3ab0c1bc, ASSIGN}] 2023-09-06T20:35:32,265 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=0489809eda53d4f420cc714a3ab0c1bc, ASSIGN 2023-09-06T20:35:32,266 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=0489809eda53d4f420cc714a3ab0c1bc, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,46803,1694032527898; forceNewPlan=false, retain=false 2023-09-06T20:35:32,417 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=0489809eda53d4f420cc714a3ab0c1bc, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:32,422 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 0489809eda53d4f420cc714a3ab0c1bc, server=jenkins-hbase3.apache.org,46803,1694032527898}] 2023-09-06T20:35:32,582 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(130): Open hbase:namespace,,1694032531333.0489809eda53d4f420cc714a3ab0c1bc. 2023-09-06T20:35:32,582 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7254): Opening region: {ENCODED => 0489809eda53d4f420cc714a3ab0c1bc, NAME => 'hbase:namespace,,1694032531333.0489809eda53d4f420cc714a3ab0c1bc.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:35:32,583 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:35:32,584 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 0489809eda53d4f420cc714a3ab0c1bc 2023-09-06T20:35:32,584 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(888): Instantiated hbase:namespace,,1694032531333.0489809eda53d4f420cc714a3ab0c1bc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:35:32,584 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7296): checking encryption for 0489809eda53d4f420cc714a3ab0c1bc 2023-09-06T20:35:32,584 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7299): checking classloading for 0489809eda53d4f420cc714a3ab0c1bc 2023-09-06T20:35:32,587 INFO [StoreOpener-0489809eda53d4f420cc714a3ab0c1bc-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 0489809eda53d4f420cc714a3ab0c1bc 2023-09-06T20:35:32,589 INFO [StoreOpener-0489809eda53d4f420cc714a3ab0c1bc-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0489809eda53d4f420cc714a3ab0c1bc columnFamilyName info 2023-09-06T20:35:32,589 DEBUG [StoreOpener-0489809eda53d4f420cc714a3ab0c1bc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:32,590 INFO [StoreOpener-0489809eda53d4f420cc714a3ab0c1bc-1 {}] regionserver.HStore(324): Store=0489809eda53d4f420cc714a3ab0c1bc/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:35:32,592 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/hbase/namespace/0489809eda53d4f420cc714a3ab0c1bc 2023-09-06T20:35:32,592 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/hbase/namespace/0489809eda53d4f420cc714a3ab0c1bc 2023-09-06T20:35:32,596 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1076): writing seq id for 0489809eda53d4f420cc714a3ab0c1bc 2023-09-06T20:35:32,599 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/hbase/namespace/0489809eda53d4f420cc714a3ab0c1bc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:35:32,600 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1093): Opened 0489809eda53d4f420cc714a3ab0c1bc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10808408800, jitterRate=0.0066115111112594604}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-06T20:35:32,601 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(992): Region open journal for 0489809eda53d4f420cc714a3ab0c1bc: 2023-09-06T20:35:32,603 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:namespace,,1694032531333.0489809eda53d4f420cc714a3ab0c1bc., pid=6, masterSystemTime=1694032532575 2023-09-06T20:35:32,606 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:namespace,,1694032531333.0489809eda53d4f420cc714a3ab0c1bc. 2023-09-06T20:35:32,606 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(158): Opened hbase:namespace,,1694032531333.0489809eda53d4f420cc714a3ab0c1bc. 2023-09-06T20:35:32,607 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=0489809eda53d4f420cc714a3ab0c1bc, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:32,613 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=6, resume processing ppid=5 2023-09-06T20:35:32,613 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 0489809eda53d4f420cc714a3ab0c1bc, server=jenkins-hbase3.apache.org,46803,1694032527898 in 188 msec 2023-09-06T20:35:32,617 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=5, resume processing ppid=4 2023-09-06T20:35:32,617 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=0489809eda53d4f420cc714a3ab0c1bc, ASSIGN in 351 msec 2023-09-06T20:35:32,618 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:35:32,618 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032532618"}]},"ts":"1694032532618"} 2023-09-06T20:35:32,624 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2023-09-06T20:35:32,627 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:35:32,629 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 1.2920 sec 2023-09-06T20:35:32,648 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Set watcher on znode that does not yet exist, /1/namespace 2023-09-06T20:35:32,651 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/namespace 2023-09-06T20:35:32,651 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:46803-0x100eefe52de0001, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:35:32,651 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:35:32,708 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2023-09-06T20:35:32,723 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-09-06T20:35:32,728 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 28 msec 2023-09-06T20:35:32,734 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2023-09-06T20:35:32,744 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-09-06T20:35:32,749 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 14 msec 2023-09-06T20:35:32,759 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/default 2023-09-06T20:35:32,764 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/hbase 2023-09-06T20:35:32,764 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1193): Master has completed initialization 4.747sec 2023-09-06T20:35:32,766 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2023-09-06T20:35:32,767 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2023-09-06T20:35:32,769 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2023-09-06T20:35:32,769 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2023-09-06T20:35:32,770 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2023-09-06T20:35:32,771 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,42111,1694032524831-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-09-06T20:35:32,772 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,42111,1694032524831-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2023-09-06T20:35:32,785 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1296): Balancer post startup initialization complete, took 0 seconds 2023-09-06T20:35:32,787 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2023-09-06T20:35:32,799 DEBUG [Listener at localhost/32815 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x045339b4 to 127.0.0.1:58778 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:35:32,803 DEBUG [Listener at localhost/32815 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18a7659f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:35:32,807 DEBUG [Listener at localhost/32815 {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2023-09-06T20:35:32,807 DEBUG [Listener at localhost/32815 {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2023-09-06T20:35:32,823 DEBUG [hconnection-0x12ffd531-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:35:32,835 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:51450, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:35:32,847 INFO [Listener at localhost/32815 {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=jenkins-hbase3.apache.org,42111,1694032524831 2023-09-06T20:35:32,847 INFO [Listener at localhost/32815 {}] hbase.HBaseTestingUtility(2786): Starting mini mapreduce cluster... 2023-09-06T20:35:32,847 INFO [Listener at localhost/32815 {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/test.cache.data in system properties and HBase conf 2023-09-06T20:35:32,847 INFO [Listener at localhost/32815 {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/hadoop.tmp.dir in system properties and HBase conf 2023-09-06T20:35:32,847 INFO [Listener at localhost/32815 {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/hadoop.log.dir in system properties and HBase conf 2023-09-06T20:35:32,847 INFO [Listener at localhost/32815 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/mapreduce.cluster.local.dir in system properties and HBase conf 2023-09-06T20:35:32,847 INFO [Listener at localhost/32815 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-09-06T20:35:32,847 INFO [Listener at localhost/32815 {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-09-06T20:35:32,847 INFO [Listener at localhost/32815 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-09-06T20:35:32,848 INFO [Listener at localhost/32815 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-09-06T20:35:32,848 INFO [Listener at localhost/32815 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-09-06T20:35:32,848 INFO [Listener at localhost/32815 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-09-06T20:35:32,848 INFO [Listener at localhost/32815 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-09-06T20:35:32,848 INFO [Listener at localhost/32815 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-09-06T20:35:32,848 INFO [Listener at localhost/32815 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-09-06T20:35:32,848 INFO [Listener at localhost/32815 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/dfs.journalnode.edits.dir in system properties and HBase conf 2023-09-06T20:35:32,849 INFO [Listener at localhost/32815 {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-09-06T20:35:32,849 INFO [Listener at localhost/32815 {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/nfs.dump.dir in system properties and HBase conf 2023-09-06T20:35:32,849 INFO [Listener at localhost/32815 {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/java.io.tmpdir in system properties and HBase conf 2023-09-06T20:35:32,849 INFO [Listener at localhost/32815 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/dfs.journalnode.edits.dir in system properties and HBase conf 2023-09-06T20:35:32,849 INFO [Listener at localhost/32815 {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-09-06T20:35:32,849 INFO [Listener at localhost/32815 {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2023-09-06T20:35:34,293 WARN [Thread-249 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-09-06T20:35:34,660 INFO [Thread-249 {}] log.Slf4jLog(67): jetty-6.1.26 2023-09-06T20:35:34,666 INFO [Thread-249 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/jobhistory to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_45455_jobhistory____y4103h/webapp 2023-09-06T20:35:34,834 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices as a root resource class 2023-09-06T20:35:34,834 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver as a provider class 2023-09-06T20:35:34,834 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-09-06T20:35:34,837 INFO [Thread-249 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-09-06T20:35:34,919 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:35:35,057 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:35:35,420 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices to GuiceManagedComponentProvider with the scope "PerRequest" 2023-09-06T20:35:35,460 INFO [Thread-249 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:45455 2023-09-06T20:35:35,789 WARN [Listener at jenkins-hbase3.apache.org/33211 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-09-06T20:35:35,868 WARN [Listener at jenkins-hbase3.apache.org/33211 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-09-06T20:35:35,869 INFO [Listener at jenkins-hbase3.apache.org/33211 {}] log.Slf4jLog(67): jetty-6.1.26 2023-09-06T20:35:35,874 INFO [Listener at jenkins-hbase3.apache.org/33211 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/cluster to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_45815_cluster____iz3p54/webapp 2023-09-06T20:35:35,977 INFO [Listener at jenkins-hbase3.apache.org/33211 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver as a provider class 2023-09-06T20:35:35,977 INFO [Listener at jenkins-hbase3.apache.org/33211 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices as a root resource class 2023-09-06T20:35:35,977 INFO [Listener at jenkins-hbase3.apache.org/33211 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-09-06T20:35:35,978 INFO [Listener at jenkins-hbase3.apache.org/33211 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-09-06T20:35:35,990 INFO [Listener at jenkins-hbase3.apache.org/33211 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:35:36,172 INFO [Listener at jenkins-hbase3.apache.org/33211 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:35:36,285 INFO [Listener at jenkins-hbase3.apache.org/33211 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:35:36,293 INFO [Listener at jenkins-hbase3.apache.org/33211 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:45815 2023-09-06T20:35:36,469 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-09-06T20:35:36,595 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2023-09-06T20:35:36,595 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2023-09-06T20:35:36,597 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2023-09-06T20:35:36,597 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2023-09-06T20:35:36,598 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:35:36,598 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver Metrics about HBase RegionObservers 2023-09-06T20:35:36,598 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-09-06T20:35:36,598 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2023-09-06T20:35:36,598 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2023-09-06T20:35:36,598 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2023-09-06T20:35:36,602 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2023-09-06T20:35:36,603 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2023-09-06T20:35:36,846 WARN [Listener at jenkins-hbase3.apache.org/38669 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-09-06T20:35:36,868 WARN [Listener at jenkins-hbase3.apache.org/38669 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-09-06T20:35:36,869 INFO [Listener at jenkins-hbase3.apache.org/38669 {}] log.Slf4jLog(67): jetty-6.1.26 2023-09-06T20:35:36,875 INFO [Listener at jenkins-hbase3.apache.org/38669 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_39595_node____r7yxhm/webapp 2023-09-06T20:35:36,928 INFO [Listener at jenkins-hbase3.apache.org/38669 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-09-06T20:35:36,928 INFO [Listener at jenkins-hbase3.apache.org/38669 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-09-06T20:35:36,928 INFO [Listener at jenkins-hbase3.apache.org/38669 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-09-06T20:35:36,928 INFO [Listener at jenkins-hbase3.apache.org/38669 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-09-06T20:35:36,936 INFO [Listener at jenkins-hbase3.apache.org/38669 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:35:36,957 INFO [Listener at jenkins-hbase3.apache.org/38669 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:35:37,037 INFO [Listener at jenkins-hbase3.apache.org/38669 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:35:37,048 INFO [Listener at jenkins-hbase3.apache.org/38669 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:39595 2023-09-06T20:35:37,190 WARN [Listener at jenkins-hbase3.apache.org/42325 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-09-06T20:35:37,199 WARN [Listener at jenkins-hbase3.apache.org/42325 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-09-06T20:35:37,199 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] log.Slf4jLog(67): jetty-6.1.26 2023-09-06T20:35:37,207 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_40923_node____.x6lqjn/webapp 2023-09-06T20:35:37,265 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-09-06T20:35:37,266 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-09-06T20:35:37,266 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-09-06T20:35:37,266 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-09-06T20:35:37,276 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:35:37,294 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:35:37,356 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:35:37,364 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:40923 2023-09-06T20:35:37,394 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] hbase.HBaseTestingUtility(2821): Mini mapreduce cluster started 2023-09-06T20:35:37,394 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] backup.TestBackupBase(301): ROOTDIR hdfs://localhost:42649/backupUT 2023-09-06T20:35:37,407 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:35:37,411 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:39620, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:35:37,421 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'ns1'} 2023-09-06T20:35:37,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] procedure2.ProcedureExecutor(1032): Stored pid=9, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns1 2023-09-06T20:35:37,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=9 2023-09-06T20:35:37,449 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-09-06T20:35:37,454 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=9, state=SUCCESS; CreateNamespaceProcedure, namespace=ns1 in 31 msec 2023-09-06T20:35:37,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=9 2023-09-06T20:35:37,545 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'ns2'} 2023-09-06T20:35:37,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] procedure2.ProcedureExecutor(1032): Stored pid=10, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns2 2023-09-06T20:35:37,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=10 2023-09-06T20:35:37,557 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-09-06T20:35:37,563 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=10, state=SUCCESS; CreateNamespaceProcedure, namespace=ns2 in 17 msec 2023-09-06T20:35:37,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=10 2023-09-06T20:35:37,655 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'ns3'} 2023-09-06T20:35:37,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] procedure2.ProcedureExecutor(1032): Stored pid=11, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns3 2023-09-06T20:35:37,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=11 2023-09-06T20:35:37,670 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-09-06T20:35:37,676 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=11, state=SUCCESS; CreateNamespaceProcedure, namespace=ns3 in 18 msec 2023-09-06T20:35:37,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=11 2023-09-06T20:35:37,766 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'ns4'} 2023-09-06T20:35:37,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] procedure2.ProcedureExecutor(1032): Stored pid=12, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns4 2023-09-06T20:35:37,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=12 2023-09-06T20:35:37,780 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-09-06T20:35:37,784 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=12, state=SUCCESS; CreateNamespaceProcedure, namespace=ns4 in 16 msec 2023-09-06T20:35:37,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=12 2023-09-06T20:35:37,889 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'test-1694032537394', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:35:37,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] procedure2.ProcedureExecutor(1032): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=test-1694032537394 2023-09-06T20:35:37,893 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=test-1694032537394 execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:35:37,893 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "test-1694032537394" procId is: 13 2023-09-06T20:35:37,894 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:37,895 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=test-1694032537394 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:35:37,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-09-06T20:35:37,931 INFO [RegionOpenAndInit-test-1694032537394-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 7595eee2e2ac3713edf2a8ba161cb7f5, NAME => 'test-1694032537394,,1694032537883.7595eee2e2ac3713edf2a8ba161cb7f5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1694032537394', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0 2023-09-06T20:35:37,946 DEBUG [RegionOpenAndInit-test-1694032537394-pool-0 {}] regionserver.HRegion(888): Instantiated test-1694032537394,,1694032537883.7595eee2e2ac3713edf2a8ba161cb7f5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:35:37,946 DEBUG [RegionOpenAndInit-test-1694032537394-pool-0 {}] regionserver.HRegion(1668): Closing 7595eee2e2ac3713edf2a8ba161cb7f5, disabling compactions & flushes 2023-09-06T20:35:37,946 INFO [RegionOpenAndInit-test-1694032537394-pool-0 {}] regionserver.HRegion(1690): Closing region test-1694032537394,,1694032537883.7595eee2e2ac3713edf2a8ba161cb7f5. 2023-09-06T20:35:37,946 DEBUG [RegionOpenAndInit-test-1694032537394-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1694032537394,,1694032537883.7595eee2e2ac3713edf2a8ba161cb7f5. 2023-09-06T20:35:37,946 DEBUG [RegionOpenAndInit-test-1694032537394-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on test-1694032537394,,1694032537883.7595eee2e2ac3713edf2a8ba161cb7f5. after waiting 0 ms 2023-09-06T20:35:37,946 DEBUG [RegionOpenAndInit-test-1694032537394-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region test-1694032537394,,1694032537883.7595eee2e2ac3713edf2a8ba161cb7f5. 2023-09-06T20:35:37,946 INFO [RegionOpenAndInit-test-1694032537394-pool-0 {}] regionserver.HRegion(1909): Closed test-1694032537394,,1694032537883.7595eee2e2ac3713edf2a8ba161cb7f5. 2023-09-06T20:35:37,946 DEBUG [RegionOpenAndInit-test-1694032537394-pool-0 {}] regionserver.HRegion(1622): Region close journal for 7595eee2e2ac3713edf2a8ba161cb7f5: 2023-09-06T20:35:37,948 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=test-1694032537394 execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:35:37,948 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"test-1694032537394,,1694032537883.7595eee2e2ac3713edf2a8ba161cb7f5.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1694032537948"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032537948"}]},"ts":"1694032537948"} 2023-09-06T20:35:37,951 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:35:37,952 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=test-1694032537394 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:35:37,953 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1694032537394","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032537952"}]},"ts":"1694032537952"} 2023-09-06T20:35:37,955 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1694032537394, state=ENABLING in hbase:meta 2023-09-06T20:35:37,961 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1694032537394, region=7595eee2e2ac3713edf2a8ba161cb7f5, ASSIGN}] 2023-09-06T20:35:37,963 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1694032537394, region=7595eee2e2ac3713edf2a8ba161cb7f5, ASSIGN 2023-09-06T20:35:37,964 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=test-1694032537394, region=7595eee2e2ac3713edf2a8ba161cb7f5, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,46803,1694032527898; forceNewPlan=false, retain=false 2023-09-06T20:35:38,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-09-06T20:35:38,115 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=7595eee2e2ac3713edf2a8ba161cb7f5, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:38,119 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE; OpenRegionProcedure 7595eee2e2ac3713edf2a8ba161cb7f5, server=jenkins-hbase3.apache.org,46803,1694032527898}] 2023-09-06T20:35:38,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-09-06T20:35:38,279 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(130): Open test-1694032537394,,1694032537883.7595eee2e2ac3713edf2a8ba161cb7f5. 2023-09-06T20:35:38,279 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7254): Opening region: {ENCODED => 7595eee2e2ac3713edf2a8ba161cb7f5, NAME => 'test-1694032537394,,1694032537883.7595eee2e2ac3713edf2a8ba161cb7f5.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:35:38,280 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:35:38,280 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1694032537394 7595eee2e2ac3713edf2a8ba161cb7f5 2023-09-06T20:35:38,280 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(888): Instantiated test-1694032537394,,1694032537883.7595eee2e2ac3713edf2a8ba161cb7f5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:35:38,280 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7296): checking encryption for 7595eee2e2ac3713edf2a8ba161cb7f5 2023-09-06T20:35:38,281 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7299): checking classloading for 7595eee2e2ac3713edf2a8ba161cb7f5 2023-09-06T20:35:38,283 INFO [StoreOpener-7595eee2e2ac3713edf2a8ba161cb7f5-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 7595eee2e2ac3713edf2a8ba161cb7f5 2023-09-06T20:35:38,285 INFO [StoreOpener-7595eee2e2ac3713edf2a8ba161cb7f5-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7595eee2e2ac3713edf2a8ba161cb7f5 columnFamilyName f 2023-09-06T20:35:38,285 DEBUG [StoreOpener-7595eee2e2ac3713edf2a8ba161cb7f5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:38,287 INFO [StoreOpener-7595eee2e2ac3713edf2a8ba161cb7f5-1 {}] regionserver.HStore(324): Store=7595eee2e2ac3713edf2a8ba161cb7f5/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:35:38,288 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/default/test-1694032537394/7595eee2e2ac3713edf2a8ba161cb7f5 2023-09-06T20:35:38,288 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/default/test-1694032537394/7595eee2e2ac3713edf2a8ba161cb7f5 2023-09-06T20:35:38,292 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1076): writing seq id for 7595eee2e2ac3713edf2a8ba161cb7f5 2023-09-06T20:35:38,298 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/default/test-1694032537394/7595eee2e2ac3713edf2a8ba161cb7f5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:35:38,298 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): Opened 7595eee2e2ac3713edf2a8ba161cb7f5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10415827680, jitterRate=-0.029950454831123352}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-06T20:35:38,300 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(992): Region open journal for 7595eee2e2ac3713edf2a8ba161cb7f5: 2023-09-06T20:35:38,301 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2556): Post open deploy tasks for test-1694032537394,,1694032537883.7595eee2e2ac3713edf2a8ba161cb7f5., pid=15, masterSystemTime=1694032538271 2023-09-06T20:35:38,304 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2583): Finished post open deploy task for test-1694032537394,,1694032537883.7595eee2e2ac3713edf2a8ba161cb7f5. 2023-09-06T20:35:38,304 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(158): Opened test-1694032537394,,1694032537883.7595eee2e2ac3713edf2a8ba161cb7f5. 2023-09-06T20:35:38,305 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=7595eee2e2ac3713edf2a8ba161cb7f5, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:38,310 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=15, resume processing ppid=14 2023-09-06T20:35:38,310 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=15, ppid=14, state=SUCCESS; OpenRegionProcedure 7595eee2e2ac3713edf2a8ba161cb7f5, server=jenkins-hbase3.apache.org,46803,1694032527898 in 189 msec 2023-09-06T20:35:38,313 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=14, resume processing ppid=13 2023-09-06T20:35:38,313 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=14, ppid=13, state=SUCCESS; TransitRegionStateProcedure table=test-1694032537394, region=7595eee2e2ac3713edf2a8ba161cb7f5, ASSIGN in 349 msec 2023-09-06T20:35:38,314 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=test-1694032537394 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:35:38,314 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1694032537394","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032538314"}]},"ts":"1694032538314"} 2023-09-06T20:35:38,316 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1694032537394, state=ENABLED in hbase:meta 2023-09-06T20:35:38,320 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=test-1694032537394 execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:35:38,322 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=13, state=SUCCESS; CreateTableProcedure table=test-1694032537394 in 431 msec 2023-09-06T20:35:38,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-09-06T20:35:38,507 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:test-1694032537394, procId: 13 completed 2023-09-06T20:35:38,509 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x7705fa08 to 127.0.0.1:58778 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:35:38,514 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32525a74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:35:38,516 DEBUG [hconnection-0x507ce415-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:35:38,520 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:51460, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:35:38,526 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46803 {}] regionserver.HRegion(8210): writing data to region test-1694032537394,,1694032537883.7595eee2e2ac3713edf2a8ba161cb7f5. with WAL disabled. Data may be lost in the event of a crash. 2023-09-06T20:35:38,629 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'ns2:test-16940325373941', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:35:38,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] procedure2.ProcedureExecutor(1032): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns2:test-16940325373941 2023-09-06T20:35:38,636 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16940325373941 execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:35:38,636 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:38,636 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns2" qualifier: "test-16940325373941" procId is: 16 2023-09-06T20:35:38,637 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns2:test-16940325373941 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:35:38,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-09-06T20:35:38,653 INFO [RegionOpenAndInit-ns2:test-16940325373941-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 9e465118ef7ff0106801ef31e1e1a961, NAME => 'ns2:test-16940325373941,,1694032538629.9e465118ef7ff0106801ef31e1e1a961.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-16940325373941', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0 2023-09-06T20:35:38,671 DEBUG [RegionOpenAndInit-ns2:test-16940325373941-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:test-16940325373941,,1694032538629.9e465118ef7ff0106801ef31e1e1a961.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:35:38,672 DEBUG [RegionOpenAndInit-ns2:test-16940325373941-pool-0 {}] regionserver.HRegion(1668): Closing 9e465118ef7ff0106801ef31e1e1a961, disabling compactions & flushes 2023-09-06T20:35:38,672 INFO [RegionOpenAndInit-ns2:test-16940325373941-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:test-16940325373941,,1694032538629.9e465118ef7ff0106801ef31e1e1a961. 2023-09-06T20:35:38,672 DEBUG [RegionOpenAndInit-ns2:test-16940325373941-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16940325373941,,1694032538629.9e465118ef7ff0106801ef31e1e1a961. 2023-09-06T20:35:38,672 DEBUG [RegionOpenAndInit-ns2:test-16940325373941-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16940325373941,,1694032538629.9e465118ef7ff0106801ef31e1e1a961. after waiting 0 ms 2023-09-06T20:35:38,672 DEBUG [RegionOpenAndInit-ns2:test-16940325373941-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16940325373941,,1694032538629.9e465118ef7ff0106801ef31e1e1a961. 2023-09-06T20:35:38,672 INFO [RegionOpenAndInit-ns2:test-16940325373941-pool-0 {}] regionserver.HRegion(1909): Closed ns2:test-16940325373941,,1694032538629.9e465118ef7ff0106801ef31e1e1a961. 2023-09-06T20:35:38,672 DEBUG [RegionOpenAndInit-ns2:test-16940325373941-pool-0 {}] regionserver.HRegion(1622): Region close journal for 9e465118ef7ff0106801ef31e1e1a961: 2023-09-06T20:35:38,673 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns2:test-16940325373941 execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:35:38,674 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:test-16940325373941,,1694032538629.9e465118ef7ff0106801ef31e1e1a961.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1694032538673"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032538673"}]},"ts":"1694032538673"} 2023-09-06T20:35:38,676 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:35:38,677 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns2:test-16940325373941 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:35:38,677 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16940325373941","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032538677"}]},"ts":"1694032538677"} 2023-09-06T20:35:38,679 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16940325373941, state=ENABLING in hbase:meta 2023-09-06T20:35:38,683 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16940325373941, region=9e465118ef7ff0106801ef31e1e1a961, ASSIGN}] 2023-09-06T20:35:38,685 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16940325373941, region=9e465118ef7ff0106801ef31e1e1a961, ASSIGN 2023-09-06T20:35:38,686 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:test-16940325373941, region=9e465118ef7ff0106801ef31e1e1a961, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,46803,1694032527898; forceNewPlan=false, retain=false 2023-09-06T20:35:38,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-09-06T20:35:38,838 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=9e465118ef7ff0106801ef31e1e1a961, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:38,841 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE; OpenRegionProcedure 9e465118ef7ff0106801ef31e1e1a961, server=jenkins-hbase3.apache.org,46803,1694032527898}] 2023-09-06T20:35:38,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-09-06T20:35:38,999 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(130): Open ns2:test-16940325373941,,1694032538629.9e465118ef7ff0106801ef31e1e1a961. 2023-09-06T20:35:38,999 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7254): Opening region: {ENCODED => 9e465118ef7ff0106801ef31e1e1a961, NAME => 'ns2:test-16940325373941,,1694032538629.9e465118ef7ff0106801ef31e1e1a961.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:35:38,999 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:35:38,999 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16940325373941 9e465118ef7ff0106801ef31e1e1a961 2023-09-06T20:35:39,000 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(888): Instantiated ns2:test-16940325373941,,1694032538629.9e465118ef7ff0106801ef31e1e1a961.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:35:39,000 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7296): checking encryption for 9e465118ef7ff0106801ef31e1e1a961 2023-09-06T20:35:39,000 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7299): checking classloading for 9e465118ef7ff0106801ef31e1e1a961 2023-09-06T20:35:39,002 INFO [StoreOpener-9e465118ef7ff0106801ef31e1e1a961-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 9e465118ef7ff0106801ef31e1e1a961 2023-09-06T20:35:39,004 INFO [StoreOpener-9e465118ef7ff0106801ef31e1e1a961-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9e465118ef7ff0106801ef31e1e1a961 columnFamilyName f 2023-09-06T20:35:39,004 DEBUG [StoreOpener-9e465118ef7ff0106801ef31e1e1a961-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:39,005 INFO [StoreOpener-9e465118ef7ff0106801ef31e1e1a961-1 {}] regionserver.HStore(324): Store=9e465118ef7ff0106801ef31e1e1a961/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:35:39,006 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/ns2/test-16940325373941/9e465118ef7ff0106801ef31e1e1a961 2023-09-06T20:35:39,006 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/ns2/test-16940325373941/9e465118ef7ff0106801ef31e1e1a961 2023-09-06T20:35:39,010 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1076): writing seq id for 9e465118ef7ff0106801ef31e1e1a961 2023-09-06T20:35:39,015 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/ns2/test-16940325373941/9e465118ef7ff0106801ef31e1e1a961/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:35:39,016 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): Opened 9e465118ef7ff0106801ef31e1e1a961; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11963095680, jitterRate=0.11415010690689087}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-06T20:35:39,017 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(992): Region open journal for 9e465118ef7ff0106801ef31e1e1a961: 2023-09-06T20:35:39,018 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2556): Post open deploy tasks for ns2:test-16940325373941,,1694032538629.9e465118ef7ff0106801ef31e1e1a961., pid=18, masterSystemTime=1694032538993 2023-09-06T20:35:39,020 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2583): Finished post open deploy task for ns2:test-16940325373941,,1694032538629.9e465118ef7ff0106801ef31e1e1a961. 2023-09-06T20:35:39,021 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(158): Opened ns2:test-16940325373941,,1694032538629.9e465118ef7ff0106801ef31e1e1a961. 2023-09-06T20:35:39,021 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=9e465118ef7ff0106801ef31e1e1a961, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:39,027 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=18, resume processing ppid=17 2023-09-06T20:35:39,027 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=18, ppid=17, state=SUCCESS; OpenRegionProcedure 9e465118ef7ff0106801ef31e1e1a961, server=jenkins-hbase3.apache.org,46803,1694032527898 in 183 msec 2023-09-06T20:35:39,029 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=17, resume processing ppid=16 2023-09-06T20:35:39,030 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=17, ppid=16, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-16940325373941, region=9e465118ef7ff0106801ef31e1e1a961, ASSIGN in 344 msec 2023-09-06T20:35:39,030 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns2:test-16940325373941 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:35:39,031 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16940325373941","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032539030"}]},"ts":"1694032539030"} 2023-09-06T20:35:39,033 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16940325373941, state=ENABLED in hbase:meta 2023-09-06T20:35:39,036 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16940325373941 execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:35:39,038 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=16, state=SUCCESS; CreateTableProcedure table=ns2:test-16940325373941 in 407 msec 2023-09-06T20:35:39,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-09-06T20:35:39,244 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: ns2:test-16940325373941, procId: 16 completed 2023-09-06T20:35:39,248 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46803 {}] regionserver.HRegion(8210): writing data to region ns2:test-16940325373941,,1694032538629.9e465118ef7ff0106801ef31e1e1a961. with WAL disabled. Data may be lost in the event of a crash. 2023-09-06T20:35:39,341 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'ns3:test-16940325373942', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:35:39,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] procedure2.ProcedureExecutor(1032): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns3:test-16940325373942 2023-09-06T20:35:39,346 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16940325373942 execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:35:39,346 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:39,346 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns3" qualifier: "test-16940325373942" procId is: 19 2023-09-06T20:35:39,347 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns3:test-16940325373942 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:35:39,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-09-06T20:35:39,366 INFO [RegionOpenAndInit-ns3:test-16940325373942-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 280ad1b518a781a3a7bb41de290d511c, NAME => 'ns3:test-16940325373942,,1694032539341.280ad1b518a781a3a7bb41de290d511c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-16940325373942', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0 2023-09-06T20:35:39,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-09-06T20:35:39,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-09-06T20:35:39,788 DEBUG [RegionOpenAndInit-ns3:test-16940325373942-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:test-16940325373942,,1694032539341.280ad1b518a781a3a7bb41de290d511c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:35:39,788 DEBUG [RegionOpenAndInit-ns3:test-16940325373942-pool-0 {}] regionserver.HRegion(1668): Closing 280ad1b518a781a3a7bb41de290d511c, disabling compactions & flushes 2023-09-06T20:35:39,788 INFO [RegionOpenAndInit-ns3:test-16940325373942-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:test-16940325373942,,1694032539341.280ad1b518a781a3a7bb41de290d511c. 2023-09-06T20:35:39,788 DEBUG [RegionOpenAndInit-ns3:test-16940325373942-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16940325373942,,1694032539341.280ad1b518a781a3a7bb41de290d511c. 2023-09-06T20:35:39,788 DEBUG [RegionOpenAndInit-ns3:test-16940325373942-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16940325373942,,1694032539341.280ad1b518a781a3a7bb41de290d511c. after waiting 0 ms 2023-09-06T20:35:39,788 DEBUG [RegionOpenAndInit-ns3:test-16940325373942-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16940325373942,,1694032539341.280ad1b518a781a3a7bb41de290d511c. 2023-09-06T20:35:39,788 INFO [RegionOpenAndInit-ns3:test-16940325373942-pool-0 {}] regionserver.HRegion(1909): Closed ns3:test-16940325373942,,1694032539341.280ad1b518a781a3a7bb41de290d511c. 2023-09-06T20:35:39,788 DEBUG [RegionOpenAndInit-ns3:test-16940325373942-pool-0 {}] regionserver.HRegion(1622): Region close journal for 280ad1b518a781a3a7bb41de290d511c: 2023-09-06T20:35:39,790 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns3:test-16940325373942 execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:35:39,790 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:test-16940325373942,,1694032539341.280ad1b518a781a3a7bb41de290d511c.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1694032539790"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032539790"}]},"ts":"1694032539790"} 2023-09-06T20:35:39,792 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:35:39,794 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns3:test-16940325373942 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:35:39,794 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16940325373942","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032539794"}]},"ts":"1694032539794"} 2023-09-06T20:35:39,796 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16940325373942, state=ENABLING in hbase:meta 2023-09-06T20:35:39,801 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16940325373942, region=280ad1b518a781a3a7bb41de290d511c, ASSIGN}] 2023-09-06T20:35:39,803 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16940325373942, region=280ad1b518a781a3a7bb41de290d511c, ASSIGN 2023-09-06T20:35:39,804 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:test-16940325373942, region=280ad1b518a781a3a7bb41de290d511c, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,46803,1694032527898; forceNewPlan=false, retain=false 2023-09-06T20:35:39,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-09-06T20:35:39,956 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=280ad1b518a781a3a7bb41de290d511c, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:39,958 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE; OpenRegionProcedure 280ad1b518a781a3a7bb41de290d511c, server=jenkins-hbase3.apache.org,46803,1694032527898}] 2023-09-06T20:35:40,116 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(130): Open ns3:test-16940325373942,,1694032539341.280ad1b518a781a3a7bb41de290d511c. 2023-09-06T20:35:40,117 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7254): Opening region: {ENCODED => 280ad1b518a781a3a7bb41de290d511c, NAME => 'ns3:test-16940325373942,,1694032539341.280ad1b518a781a3a7bb41de290d511c.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:35:40,117 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:35:40,117 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16940325373942 280ad1b518a781a3a7bb41de290d511c 2023-09-06T20:35:40,117 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(888): Instantiated ns3:test-16940325373942,,1694032539341.280ad1b518a781a3a7bb41de290d511c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:35:40,117 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7296): checking encryption for 280ad1b518a781a3a7bb41de290d511c 2023-09-06T20:35:40,117 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7299): checking classloading for 280ad1b518a781a3a7bb41de290d511c 2023-09-06T20:35:40,119 INFO [StoreOpener-280ad1b518a781a3a7bb41de290d511c-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 280ad1b518a781a3a7bb41de290d511c 2023-09-06T20:35:40,121 INFO [StoreOpener-280ad1b518a781a3a7bb41de290d511c-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 280ad1b518a781a3a7bb41de290d511c columnFamilyName f 2023-09-06T20:35:40,121 DEBUG [StoreOpener-280ad1b518a781a3a7bb41de290d511c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:40,122 INFO [StoreOpener-280ad1b518a781a3a7bb41de290d511c-1 {}] regionserver.HStore(324): Store=280ad1b518a781a3a7bb41de290d511c/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:35:40,123 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/ns3/test-16940325373942/280ad1b518a781a3a7bb41de290d511c 2023-09-06T20:35:40,124 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/ns3/test-16940325373942/280ad1b518a781a3a7bb41de290d511c 2023-09-06T20:35:40,127 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1076): writing seq id for 280ad1b518a781a3a7bb41de290d511c 2023-09-06T20:35:40,130 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/ns3/test-16940325373942/280ad1b518a781a3a7bb41de290d511c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:35:40,131 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): Opened 280ad1b518a781a3a7bb41de290d511c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11693021440, jitterRate=0.088997483253479}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-06T20:35:40,132 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(992): Region open journal for 280ad1b518a781a3a7bb41de290d511c: 2023-09-06T20:35:40,134 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2556): Post open deploy tasks for ns3:test-16940325373942,,1694032539341.280ad1b518a781a3a7bb41de290d511c., pid=21, masterSystemTime=1694032540111 2023-09-06T20:35:40,136 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2583): Finished post open deploy task for ns3:test-16940325373942,,1694032539341.280ad1b518a781a3a7bb41de290d511c. 2023-09-06T20:35:40,136 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(158): Opened ns3:test-16940325373942,,1694032539341.280ad1b518a781a3a7bb41de290d511c. 2023-09-06T20:35:40,137 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=280ad1b518a781a3a7bb41de290d511c, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:40,141 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=21, resume processing ppid=20 2023-09-06T20:35:40,142 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=21, ppid=20, state=SUCCESS; OpenRegionProcedure 280ad1b518a781a3a7bb41de290d511c, server=jenkins-hbase3.apache.org,46803,1694032527898 in 181 msec 2023-09-06T20:35:40,144 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=20, resume processing ppid=19 2023-09-06T20:35:40,144 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=20, ppid=19, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-16940325373942, region=280ad1b518a781a3a7bb41de290d511c, ASSIGN in 341 msec 2023-09-06T20:35:40,145 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns3:test-16940325373942 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:35:40,145 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16940325373942","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032540145"}]},"ts":"1694032540145"} 2023-09-06T20:35:40,148 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16940325373942, state=ENABLED in hbase:meta 2023-09-06T20:35:40,151 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16940325373942 execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:35:40,152 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=19, state=SUCCESS; CreateTableProcedure table=ns3:test-16940325373942 in 810 msec 2023-09-06T20:35:40,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-09-06T20:35:40,456 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: ns3:test-16940325373942, procId: 19 completed 2023-09-06T20:35:40,456 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns3:test-16940325373942 get assigned. Timeout = 60000ms 2023-09-06T20:35:40,458 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-06T20:35:40,463 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] hbase.HBaseTestingUtility(3581): All regions for table ns3:test-16940325373942 assigned to meta. Checking AM states. 2023-09-06T20:35:40,463 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-06T20:35:40,463 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] hbase.HBaseTestingUtility(3601): All regions for table ns3:test-16940325373942 assigned. 2023-09-06T20:35:40,466 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'ns4:test-16940325373943', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:35:40,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] procedure2.ProcedureExecutor(1032): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns4:test-16940325373943 2023-09-06T20:35:40,469 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16940325373943 execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:35:40,469 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:40,470 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns4" qualifier: "test-16940325373943" procId is: 22 2023-09-06T20:35:40,471 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns4:test-16940325373943 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:35:40,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-09-06T20:35:40,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-09-06T20:35:40,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-09-06T20:35:40,889 INFO [RegionOpenAndInit-ns4:test-16940325373943-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 67d870a84d8d83d52f8b7cef0eced24d, NAME => 'ns4:test-16940325373943,,1694032540465.67d870a84d8d83d52f8b7cef0eced24d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns4:test-16940325373943', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0 2023-09-06T20:35:41,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-09-06T20:35:41,305 DEBUG [RegionOpenAndInit-ns4:test-16940325373943-pool-0 {}] regionserver.HRegion(888): Instantiated ns4:test-16940325373943,,1694032540465.67d870a84d8d83d52f8b7cef0eced24d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:35:41,306 DEBUG [RegionOpenAndInit-ns4:test-16940325373943-pool-0 {}] regionserver.HRegion(1668): Closing 67d870a84d8d83d52f8b7cef0eced24d, disabling compactions & flushes 2023-09-06T20:35:41,306 INFO [RegionOpenAndInit-ns4:test-16940325373943-pool-0 {}] regionserver.HRegion(1690): Closing region ns4:test-16940325373943,,1694032540465.67d870a84d8d83d52f8b7cef0eced24d. 2023-09-06T20:35:41,306 DEBUG [RegionOpenAndInit-ns4:test-16940325373943-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16940325373943,,1694032540465.67d870a84d8d83d52f8b7cef0eced24d. 2023-09-06T20:35:41,306 DEBUG [RegionOpenAndInit-ns4:test-16940325373943-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16940325373943,,1694032540465.67d870a84d8d83d52f8b7cef0eced24d. after waiting 0 ms 2023-09-06T20:35:41,306 DEBUG [RegionOpenAndInit-ns4:test-16940325373943-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16940325373943,,1694032540465.67d870a84d8d83d52f8b7cef0eced24d. 2023-09-06T20:35:41,306 INFO [RegionOpenAndInit-ns4:test-16940325373943-pool-0 {}] regionserver.HRegion(1909): Closed ns4:test-16940325373943,,1694032540465.67d870a84d8d83d52f8b7cef0eced24d. 2023-09-06T20:35:41,306 DEBUG [RegionOpenAndInit-ns4:test-16940325373943-pool-0 {}] regionserver.HRegion(1622): Region close journal for 67d870a84d8d83d52f8b7cef0eced24d: 2023-09-06T20:35:41,307 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns4:test-16940325373943 execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:35:41,307 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns4:test-16940325373943,,1694032540465.67d870a84d8d83d52f8b7cef0eced24d.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1694032541307"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032541307"}]},"ts":"1694032541307"} 2023-09-06T20:35:41,310 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:35:41,311 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns4:test-16940325373943 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:35:41,311 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16940325373943","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032541311"}]},"ts":"1694032541311"} 2023-09-06T20:35:41,312 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16940325373943, state=ENABLING in hbase:meta 2023-09-06T20:35:41,318 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16940325373943, region=67d870a84d8d83d52f8b7cef0eced24d, ASSIGN}] 2023-09-06T20:35:41,320 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16940325373943, region=67d870a84d8d83d52f8b7cef0eced24d, ASSIGN 2023-09-06T20:35:41,321 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns4:test-16940325373943, region=67d870a84d8d83d52f8b7cef0eced24d, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,46803,1694032527898; forceNewPlan=false, retain=false 2023-09-06T20:35:41,473 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=67d870a84d8d83d52f8b7cef0eced24d, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:41,476 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE; OpenRegionProcedure 67d870a84d8d83d52f8b7cef0eced24d, server=jenkins-hbase3.apache.org,46803,1694032527898}] 2023-09-06T20:35:41,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-09-06T20:35:41,631 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(130): Open ns4:test-16940325373943,,1694032540465.67d870a84d8d83d52f8b7cef0eced24d. 2023-09-06T20:35:41,632 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7254): Opening region: {ENCODED => 67d870a84d8d83d52f8b7cef0eced24d, NAME => 'ns4:test-16940325373943,,1694032540465.67d870a84d8d83d52f8b7cef0eced24d.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:35:41,632 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:35:41,632 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16940325373943 67d870a84d8d83d52f8b7cef0eced24d 2023-09-06T20:35:41,632 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(888): Instantiated ns4:test-16940325373943,,1694032540465.67d870a84d8d83d52f8b7cef0eced24d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:35:41,632 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7296): checking encryption for 67d870a84d8d83d52f8b7cef0eced24d 2023-09-06T20:35:41,632 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7299): checking classloading for 67d870a84d8d83d52f8b7cef0eced24d 2023-09-06T20:35:41,634 INFO [StoreOpener-67d870a84d8d83d52f8b7cef0eced24d-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 67d870a84d8d83d52f8b7cef0eced24d 2023-09-06T20:35:41,636 INFO [StoreOpener-67d870a84d8d83d52f8b7cef0eced24d-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 67d870a84d8d83d52f8b7cef0eced24d columnFamilyName f 2023-09-06T20:35:41,636 DEBUG [StoreOpener-67d870a84d8d83d52f8b7cef0eced24d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:41,637 INFO [StoreOpener-67d870a84d8d83d52f8b7cef0eced24d-1 {}] regionserver.HStore(324): Store=67d870a84d8d83d52f8b7cef0eced24d/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:35:41,638 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/ns4/test-16940325373943/67d870a84d8d83d52f8b7cef0eced24d 2023-09-06T20:35:41,639 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/ns4/test-16940325373943/67d870a84d8d83d52f8b7cef0eced24d 2023-09-06T20:35:41,642 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1076): writing seq id for 67d870a84d8d83d52f8b7cef0eced24d 2023-09-06T20:35:41,645 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/ns4/test-16940325373943/67d870a84d8d83d52f8b7cef0eced24d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:35:41,646 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): Opened 67d870a84d8d83d52f8b7cef0eced24d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11060275200, jitterRate=0.030068397521972656}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-06T20:35:41,647 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(992): Region open journal for 67d870a84d8d83d52f8b7cef0eced24d: 2023-09-06T20:35:41,648 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2556): Post open deploy tasks for ns4:test-16940325373943,,1694032540465.67d870a84d8d83d52f8b7cef0eced24d., pid=24, masterSystemTime=1694032541627 2023-09-06T20:35:41,650 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2583): Finished post open deploy task for ns4:test-16940325373943,,1694032540465.67d870a84d8d83d52f8b7cef0eced24d. 2023-09-06T20:35:41,650 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(158): Opened ns4:test-16940325373943,,1694032540465.67d870a84d8d83d52f8b7cef0eced24d. 2023-09-06T20:35:41,651 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=67d870a84d8d83d52f8b7cef0eced24d, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:41,656 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=24, resume processing ppid=23 2023-09-06T20:35:41,656 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=24, ppid=23, state=SUCCESS; OpenRegionProcedure 67d870a84d8d83d52f8b7cef0eced24d, server=jenkins-hbase3.apache.org,46803,1694032527898 in 178 msec 2023-09-06T20:35:41,659 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=23, resume processing ppid=22 2023-09-06T20:35:41,659 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=23, ppid=22, state=SUCCESS; TransitRegionStateProcedure table=ns4:test-16940325373943, region=67d870a84d8d83d52f8b7cef0eced24d, ASSIGN in 338 msec 2023-09-06T20:35:41,660 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns4:test-16940325373943 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:35:41,660 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16940325373943","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032541660"}]},"ts":"1694032541660"} 2023-09-06T20:35:41,662 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16940325373943, state=ENABLED in hbase:meta 2023-09-06T20:35:41,665 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16940325373943 execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:35:41,667 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=22, state=SUCCESS; CreateTableProcedure table=ns4:test-16940325373943 in 1.1990 sec 2023-09-06T20:35:42,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-09-06T20:35:42,581 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: ns4:test-16940325373943, procId: 22 completed 2023-09-06T20:35:42,581 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns4:test-16940325373943 get assigned. Timeout = 60000ms 2023-09-06T20:35:42,581 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-06T20:35:42,586 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] hbase.HBaseTestingUtility(3581): All regions for table ns4:test-16940325373943 assigned to meta. Checking AM states. 2023-09-06T20:35:42,586 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-06T20:35:42,586 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] hbase.HBaseTestingUtility(3601): All regions for table ns4:test-16940325373943 assigned. 2023-09-06T20:35:42,587 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x7705fa08 to 127.0.0.1:58778 2023-09-06T20:35:42,587 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:35:42,616 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-16940325373942' 2023-09-06T20:35:42,617 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns4:test-16940325373943' 2023-09-06T20:35:42,618 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test-1694032537394' 2023-09-06T20:35:42,618 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:test-16940325373941' 2023-09-06T20:35:42,628 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] hbase.ResourceChecker(147): before: backup.TestBackupDelete#testBackupDeleteCommand Thread=584, OpenFileDescriptor=660, MaxFileDescriptor=60000, SystemLoadAverage=139, ProcessCount=167, AvailableMemoryMB=3459 2023-09-06T20:35:42,628 WARN [Listener at jenkins-hbase3.apache.org/42325 {}] hbase.ResourceChecker(130): Thread=584 is superior to 500 2023-09-06T20:35:42,630 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] backup.TestBackupDelete(86): test backup delete on a single table with data: command-line 2023-09-06T20:35:42,631 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x34ced1ef to 127.0.0.1:58778 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:35:42,638 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1247cc89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:35:42,646 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:35:42,647 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:37660, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:35:42,659 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:35:42,661 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:49106, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:35:42,671 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'backup'} 2023-09-06T20:35:42,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] procedure2.ProcedureExecutor(1032): Stored pid=25, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=backup 2023-09-06T20:35:42,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=25 2023-09-06T20:35:42,685 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-09-06T20:35:42,692 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=25, state=SUCCESS; CreateNamespaceProcedure, namespace=backup in 17 msec 2023-09-06T20:35:42,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=25 2023-09-06T20:35:42,794 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'backup:system', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:35:42,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] procedure2.ProcedureExecutor(1032): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system 2023-09-06T20:35:42,798 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:35:42,798 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:42,798 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system" procId is: 26 2023-09-06T20:35:42,799 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:35:42,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-09-06T20:35:42,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-09-06T20:35:43,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-09-06T20:35:43,219 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => c972ac6cf58538c05726e9a461c31a7f, NAME => 'backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0 2023-09-06T20:35:43,231 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:35:43,231 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1668): Closing c972ac6cf58538c05726e9a461c31a7f, disabling compactions & flushes 2023-09-06T20:35:43,231 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f. 2023-09-06T20:35:43,231 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f. 2023-09-06T20:35:43,231 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f. after waiting 0 ms 2023-09-06T20:35:43,231 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f. 2023-09-06T20:35:43,231 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1909): Closed backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f. 2023-09-06T20:35:43,232 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1622): Region close journal for c972ac6cf58538c05726e9a461c31a7f: 2023-09-06T20:35:43,233 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:35:43,234 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1694032543233"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032543233"}]},"ts":"1694032543233"} 2023-09-06T20:35:43,236 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:35:43,237 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:35:43,237 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032543237"}]},"ts":"1694032543237"} 2023-09-06T20:35:43,239 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-09-06T20:35:43,244 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=c972ac6cf58538c05726e9a461c31a7f, ASSIGN}] 2023-09-06T20:35:43,246 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=c972ac6cf58538c05726e9a461c31a7f, ASSIGN 2023-09-06T20:35:43,247 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=c972ac6cf58538c05726e9a461c31a7f, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,46803,1694032527898; forceNewPlan=false, retain=false 2023-09-06T20:35:43,398 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=c972ac6cf58538c05726e9a461c31a7f, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:43,401 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE; OpenRegionProcedure c972ac6cf58538c05726e9a461c31a7f, server=jenkins-hbase3.apache.org,46803,1694032527898}] 2023-09-06T20:35:43,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-09-06T20:35:43,557 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(130): Open backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f. 2023-09-06T20:35:43,557 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7254): Opening region: {ENCODED => c972ac6cf58538c05726e9a461c31a7f, NAME => 'backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:35:43,558 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:35:43,558 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system c972ac6cf58538c05726e9a461c31a7f 2023-09-06T20:35:43,558 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(888): Instantiated backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:35:43,558 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7296): checking encryption for c972ac6cf58538c05726e9a461c31a7f 2023-09-06T20:35:43,559 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7299): checking classloading for c972ac6cf58538c05726e9a461c31a7f 2023-09-06T20:35:43,560 INFO [StoreOpener-c972ac6cf58538c05726e9a461c31a7f-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region c972ac6cf58538c05726e9a461c31a7f 2023-09-06T20:35:43,563 INFO [StoreOpener-c972ac6cf58538c05726e9a461c31a7f-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c972ac6cf58538c05726e9a461c31a7f columnFamilyName meta 2023-09-06T20:35:43,563 DEBUG [StoreOpener-c972ac6cf58538c05726e9a461c31a7f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:43,563 INFO [StoreOpener-c972ac6cf58538c05726e9a461c31a7f-1 {}] regionserver.HStore(324): Store=c972ac6cf58538c05726e9a461c31a7f/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:35:43,564 INFO [StoreOpener-c972ac6cf58538c05726e9a461c31a7f-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region c972ac6cf58538c05726e9a461c31a7f 2023-09-06T20:35:43,565 INFO [StoreOpener-c972ac6cf58538c05726e9a461c31a7f-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c972ac6cf58538c05726e9a461c31a7f columnFamilyName session 2023-09-06T20:35:43,565 DEBUG [StoreOpener-c972ac6cf58538c05726e9a461c31a7f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:43,566 INFO [StoreOpener-c972ac6cf58538c05726e9a461c31a7f-1 {}] regionserver.HStore(324): Store=c972ac6cf58538c05726e9a461c31a7f/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:35:43,567 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f 2023-09-06T20:35:43,567 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f 2023-09-06T20:35:43,569 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-09-06T20:35:43,571 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1076): writing seq id for c972ac6cf58538c05726e9a461c31a7f 2023-09-06T20:35:43,573 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:35:43,574 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): Opened c972ac6cf58538c05726e9a461c31a7f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=12039314720, jitterRate=0.12124855816364288}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-09-06T20:35:43,575 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(992): Region open journal for c972ac6cf58538c05726e9a461c31a7f: 2023-09-06T20:35:43,576 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f., pid=28, masterSystemTime=1694032543553 2023-09-06T20:35:43,578 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f. 2023-09-06T20:35:43,578 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(158): Opened backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f. 2023-09-06T20:35:43,579 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=c972ac6cf58538c05726e9a461c31a7f, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:43,583 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=28, resume processing ppid=27 2023-09-06T20:35:43,583 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=28, ppid=27, state=SUCCESS; OpenRegionProcedure c972ac6cf58538c05726e9a461c31a7f, server=jenkins-hbase3.apache.org,46803,1694032527898 in 180 msec 2023-09-06T20:35:43,585 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=27, resume processing ppid=26 2023-09-06T20:35:43,585 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=27, ppid=26, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=c972ac6cf58538c05726e9a461c31a7f, ASSIGN in 339 msec 2023-09-06T20:35:43,586 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:35:43,586 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032543586"}]},"ts":"1694032543586"} 2023-09-06T20:35:43,588 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-09-06T20:35:43,590 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:35:43,592 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=26, state=SUCCESS; CreateTableProcedure table=backup:system in 796 msec 2023-09-06T20:35:43,789 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-09-06T20:35:43,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-09-06T20:35:43,906 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: backup:system, procId: 26 completed 2023-09-06T20:35:43,920 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'backup:system_bulk', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:35:43,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] procedure2.ProcedureExecutor(1032): Stored pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system_bulk 2023-09-06T20:35:43,924 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:35:43,924 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:43,924 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system_bulk" procId is: 29 2023-09-06T20:35:43,925 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:35:43,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-09-06T20:35:43,938 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 15cfd8c09eef178ff8687020ba736ee6, NAME => 'backup:system_bulk,,1694032543920.15cfd8c09eef178ff8687020ba736ee6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0 2023-09-06T20:35:44,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-09-06T20:35:44,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-09-06T20:35:44,349 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1694032543920.15cfd8c09eef178ff8687020ba736ee6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:35:44,349 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1668): Closing 15cfd8c09eef178ff8687020ba736ee6, disabling compactions & flushes 2023-09-06T20:35:44,349 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1694032543920.15cfd8c09eef178ff8687020ba736ee6. 2023-09-06T20:35:44,349 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1694032543920.15cfd8c09eef178ff8687020ba736ee6. 2023-09-06T20:35:44,349 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1694032543920.15cfd8c09eef178ff8687020ba736ee6. after waiting 0 ms 2023-09-06T20:35:44,349 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1694032543920.15cfd8c09eef178ff8687020ba736ee6. 2023-09-06T20:35:44,349 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1909): Closed backup:system_bulk,,1694032543920.15cfd8c09eef178ff8687020ba736ee6. 2023-09-06T20:35:44,350 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1622): Region close journal for 15cfd8c09eef178ff8687020ba736ee6: 2023-09-06T20:35:44,351 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:35:44,351 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system_bulk,,1694032543920.15cfd8c09eef178ff8687020ba736ee6.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1694032544351"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032544351"}]},"ts":"1694032544351"} 2023-09-06T20:35:44,354 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:35:44,355 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:35:44,355 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032544355"}]},"ts":"1694032544355"} 2023-09-06T20:35:44,357 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLING in hbase:meta 2023-09-06T20:35:44,361 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=15cfd8c09eef178ff8687020ba736ee6, ASSIGN}] 2023-09-06T20:35:44,363 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=15cfd8c09eef178ff8687020ba736ee6, ASSIGN 2023-09-06T20:35:44,364 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system_bulk, region=15cfd8c09eef178ff8687020ba736ee6, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,46803,1694032527898; forceNewPlan=false, retain=false 2023-09-06T20:35:44,516 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=15cfd8c09eef178ff8687020ba736ee6, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:44,518 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE; OpenRegionProcedure 15cfd8c09eef178ff8687020ba736ee6, server=jenkins-hbase3.apache.org,46803,1694032527898}] 2023-09-06T20:35:44,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-09-06T20:35:44,675 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(130): Open backup:system_bulk,,1694032543920.15cfd8c09eef178ff8687020ba736ee6. 2023-09-06T20:35:44,675 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7254): Opening region: {ENCODED => 15cfd8c09eef178ff8687020ba736ee6, NAME => 'backup:system_bulk,,1694032543920.15cfd8c09eef178ff8687020ba736ee6.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:35:44,675 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:35:44,676 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system_bulk 15cfd8c09eef178ff8687020ba736ee6 2023-09-06T20:35:44,676 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1694032543920.15cfd8c09eef178ff8687020ba736ee6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:35:44,676 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7296): checking encryption for 15cfd8c09eef178ff8687020ba736ee6 2023-09-06T20:35:44,676 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7299): checking classloading for 15cfd8c09eef178ff8687020ba736ee6 2023-09-06T20:35:44,678 INFO [StoreOpener-15cfd8c09eef178ff8687020ba736ee6-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 15cfd8c09eef178ff8687020ba736ee6 2023-09-06T20:35:44,679 INFO [StoreOpener-15cfd8c09eef178ff8687020ba736ee6-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15cfd8c09eef178ff8687020ba736ee6 columnFamilyName meta 2023-09-06T20:35:44,680 DEBUG [StoreOpener-15cfd8c09eef178ff8687020ba736ee6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:44,680 INFO [StoreOpener-15cfd8c09eef178ff8687020ba736ee6-1 {}] regionserver.HStore(324): Store=15cfd8c09eef178ff8687020ba736ee6/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:35:44,680 INFO [StoreOpener-15cfd8c09eef178ff8687020ba736ee6-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 15cfd8c09eef178ff8687020ba736ee6 2023-09-06T20:35:44,682 INFO [StoreOpener-15cfd8c09eef178ff8687020ba736ee6-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15cfd8c09eef178ff8687020ba736ee6 columnFamilyName session 2023-09-06T20:35:44,682 DEBUG [StoreOpener-15cfd8c09eef178ff8687020ba736ee6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:44,683 INFO [StoreOpener-15cfd8c09eef178ff8687020ba736ee6-1 {}] regionserver.HStore(324): Store=15cfd8c09eef178ff8687020ba736ee6/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:35:44,684 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system_bulk/15cfd8c09eef178ff8687020ba736ee6 2023-09-06T20:35:44,684 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system_bulk/15cfd8c09eef178ff8687020ba736ee6 2023-09-06T20:35:44,686 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system_bulk descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-09-06T20:35:44,688 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1076): writing seq id for 15cfd8c09eef178ff8687020ba736ee6 2023-09-06T20:35:44,690 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system_bulk/15cfd8c09eef178ff8687020ba736ee6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:35:44,691 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): Opened 15cfd8c09eef178ff8687020ba736ee6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11589035840, jitterRate=0.07931306958198547}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-09-06T20:35:44,692 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(992): Region open journal for 15cfd8c09eef178ff8687020ba736ee6: 2023-09-06T20:35:44,693 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system_bulk,,1694032543920.15cfd8c09eef178ff8687020ba736ee6., pid=31, masterSystemTime=1694032544670 2023-09-06T20:35:44,695 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system_bulk,,1694032543920.15cfd8c09eef178ff8687020ba736ee6. 2023-09-06T20:35:44,695 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(158): Opened backup:system_bulk,,1694032543920.15cfd8c09eef178ff8687020ba736ee6. 2023-09-06T20:35:44,696 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=15cfd8c09eef178ff8687020ba736ee6, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:44,700 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=31, resume processing ppid=30 2023-09-06T20:35:44,700 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=31, ppid=30, state=SUCCESS; OpenRegionProcedure 15cfd8c09eef178ff8687020ba736ee6, server=jenkins-hbase3.apache.org,46803,1694032527898 in 180 msec 2023-09-06T20:35:44,702 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=30, resume processing ppid=29 2023-09-06T20:35:44,703 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=30, ppid=29, state=SUCCESS; TransitRegionStateProcedure table=backup:system_bulk, region=15cfd8c09eef178ff8687020ba736ee6, ASSIGN in 339 msec 2023-09-06T20:35:44,703 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:35:44,704 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032544703"}]},"ts":"1694032544703"} 2023-09-06T20:35:44,705 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLED in hbase:meta 2023-09-06T20:35:44,708 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:35:44,709 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=29, state=SUCCESS; CreateTableProcedure table=backup:system_bulk in 788 msec 2023-09-06T20:35:45,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-09-06T20:35:45,033 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: backup:system_bulk, procId: 29 completed 2023-09-06T20:35:45,047 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-09-06T20:35:45,123 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:35:45,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032545123 (current time:1694032545123). 2023-09-06T20:35:45,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:35:45,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-09-06T20:35:45,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x757d9098 to 127.0.0.1:58778 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:35:45,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dfc448b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:35:45,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:35:45,137 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:37662, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:35:45,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x757d9098 to 127.0.0.1:58778 2023-09-06T20:35:45,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:35:45,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:35:45,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] procedure2.ProcedureExecutor(1032): Stored pid=32, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:35:45,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-09-06T20:35:45,154 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:35:45,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-09-06T20:35:45,159 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:35:45,161 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:35:45,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-09-06T20:35:45,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-09-06T20:35:45,572 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:35:45,575 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure c972ac6cf58538c05726e9a461c31a7f}] 2023-09-06T20:35:45,582 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure c972ac6cf58538c05726e9a461c31a7f 2023-09-06T20:35:45,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46803 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=33 2023-09-06T20:35:45,744 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f. 2023-09-06T20:35:45,746 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2819): Flushing c972ac6cf58538c05726e9a461c31a7f 2/2 column families, dataSize=45 B heapSize=632 B 2023-09-06T20:35:45,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-09-06T20:35:45,821 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/.tmp/session/020ff1bc22044c989574f67c5d443447 is 49, key is activesession:/session:c/1694032545108/Put/seqid=0 2023-09-06T20:35:45,967 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-09-06T20:35:45,967 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2023-09-06T20:35:45,968 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-16940325373942 2023-09-06T20:35:45,968 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns3_table_test-16940325373942 Metrics about Tables on a single HBase RegionServer 2023-09-06T20:35:45,969 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2023-09-06T20:35:45,969 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk Metrics about Tables on a single HBase RegionServer 2023-09-06T20:35:45,970 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-16940325373943 2023-09-06T20:35:45,970 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns4_table_test-16940325373943 Metrics about Tables on a single HBase RegionServer 2023-09-06T20:35:45,970 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1694032537394 2023-09-06T20:35:45,970 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test-1694032537394 Metrics about Tables on a single HBase RegionServer 2023-09-06T20:35:45,971 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-16940325373941 2023-09-06T20:35:45,971 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_test-16940325373941 Metrics about Tables on a single HBase RegionServer 2023-09-06T20:35:46,240 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/.tmp/session/020ff1bc22044c989574f67c5d443447 2023-09-06T20:35:46,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-09-06T20:35:46,317 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/.tmp/session/020ff1bc22044c989574f67c5d443447 as hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/020ff1bc22044c989574f67c5d443447 2023-09-06T20:35:46,326 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/020ff1bc22044c989574f67c5d443447, entries=1, sequenceid=5, filesize=4.9 K 2023-09-06T20:35:46,330 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(3022): Finished flush of dataSize ~45 B/45, heapSize ~360 B/360, currentSize=0 B/0 for c972ac6cf58538c05726e9a461c31a7f in 584ms, sequenceid=5, compaction requested=false 2023-09-06T20:35:46,330 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-09-06T20:35:46,332 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2520): Flush status journal for c972ac6cf58538c05726e9a461c31a7f: 2023-09-06T20:35:46,332 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f. for snapshot_backup_system completed. 2023-09-06T20:35:46,333 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f.' region-info for snapshot=snapshot_backup_system 2023-09-06T20:35:46,339 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:35:46,344 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-09-06T20:35:46,344 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/020ff1bc22044c989574f67c5d443447] hfiles 2023-09-06T20:35:46,344 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/020ff1bc22044c989574f67c5d443447 for snapshot=snapshot_backup_system 2023-09-06T20:35:46,362 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f. 2023-09-06T20:35:46,366 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=33 2023-09-06T20:35:46,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.HMaster(3995): Remote procedure done, pid=33 2023-09-06T20:35:46,369 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region c972ac6cf58538c05726e9a461c31a7f 2023-09-06T20:35:46,370 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure c972ac6cf58538c05726e9a461c31a7f 2023-09-06T20:35:46,376 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=33, resume processing ppid=32 2023-09-06T20:35:46,376 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=33, ppid=32, state=SUCCESS; SnapshotRegionProcedure c972ac6cf58538c05726e9a461c31a7f in 796 msec 2023-09-06T20:35:46,376 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:35:46,378 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:35:46,382 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:35:46,382 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-09-06T20:35:46,384 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/.hbase-snapshot/.tmp/snapshot_backup_system 2023-09-06T20:35:46,811 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:35:46,825 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:35:46,826 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/.hbase-snapshot/snapshot_backup_system 2023-09-06T20:35:46,828 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:35:46,828 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-09-06T20:35:46,829 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=32, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.6820 sec 2023-09-06T20:35:47,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-09-06T20:35:47,263 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] client.HBaseAdmin$TableFuture(3677): Operation: SNAPSHOT, Table Name: backup:system, procId: 32 completed 2023-09-06T20:35:47,263 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] impl.TableBackupClient(115): Backup backup_1694032542641 started at 1694032547263. 2023-09-06T20:35:47,285 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.TableBackupClient(119): Backup session backup_1694032542641 has been started. 2023-09-06T20:35:47,289 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-09-06T20:35:47,299 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-09-06T20:35:47,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-09-06T20:35:47,305 INFO [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-09-06T20:35:47,305 DEBUG [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-09-06T20:35:47,306 DEBUG [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-09-06T20:35:47,306 DEBUG [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-09-06T20:35:47,307 DEBUG [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-09-06T20:35:47,307 DEBUG [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:35:47,308 DEBUG [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:47,308 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:46803-0x100eefe52de0001, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-09-06T20:35:47,309 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:35:47,309 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:35:47,309 DEBUG [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:47,309 DEBUG [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-09-06T20:35:47,309 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-09-06T20:35:47,309 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46803-0x100eefe52de0001, quorum=127.0.0.1:58778, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-09-06T20:35:47,310 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-09-06T20:35:47,310 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:35:47,310 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-09-06T20:35:47,312 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-09-06T20:35:47,312 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-09-06T20:35:47,312 DEBUG [member: 'jenkins-hbase3.apache.org,46803,1694032527898' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-09-06T20:35:47,312 DEBUG [member: 'jenkins-hbase3.apache.org,46803,1694032527898' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-09-06T20:35:47,313 DEBUG [member: 'jenkins-hbase3.apache.org,46803,1694032527898' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-09-06T20:35:47,313 DEBUG [member: 'jenkins-hbase3.apache.org,46803,1694032527898' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-09-06T20:35:47,313 DEBUG [member: 'jenkins-hbase3.apache.org,46803,1694032527898' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,46803,1694032527898' joining acquired barrier for procedure (rolllog) in zk 2023-09-06T20:35:47,315 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:47,315 DEBUG [member: 'jenkins-hbase3.apache.org,46803,1694032527898' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-09-06T20:35:47,315 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:47,315 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:35:47,315 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:35:47,316 DEBUG [member: 'jenkins-hbase3.apache.org,46803,1694032527898' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46803-0x100eefe52de0001, quorum=127.0.0.1:58778, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-09-06T20:35:47,316 DEBUG [member: 'jenkins-hbase3.apache.org,46803,1694032527898' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-09-06T20:35:47,316 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:35:47,316 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:35:47,317 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:35:47,317 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:47,318 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:35:47,318 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,46803,1694032527898' joining acquired barrier for procedure 'rolllog' on coordinator 2023-09-06T20:35:47,318 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@226228a5[Count = 0] remaining members to acquire global barrier 2023-09-06T20:35:47,318 DEBUG [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-09-06T20:35:47,318 DEBUG [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-09-06T20:35:47,321 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:46803-0x100eefe52de0001, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-09-06T20:35:47,321 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-09-06T20:35:47,321 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-09-06T20:35:47,321 DEBUG [member: 'jenkins-hbase3.apache.org,46803,1694032527898' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-09-06T20:35:47,321 DEBUG [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:47,321 DEBUG [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-09-06T20:35:47,322 DEBUG [member: 'jenkins-hbase3.apache.org,46803,1694032527898' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-09-06T20:35:47,322 DEBUG [rs(jenkins-hbase3.apache.org,46803,1694032527898)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:47,322 INFO [rs(jenkins-hbase3.apache.org,46803,1694032527898)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1694032530354 highest: 1694032530354 on jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:47,323 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C46803%2C1694032527898.meta:.meta(num 1694032530884) roll requested 2023-09-06T20:35:47,338 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/WALs/jenkins-hbase3.apache.org,46803,1694032527898/jenkins-hbase3.apache.org%2C46803%2C1694032527898.meta.1694032547323.meta, exclude list is [], retry=0 2023-09-06T20:35:47,344 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36635,DS-c9c70a6a-ab73-4861-a2f6-0e02c09a2252,DISK] 2023-09-06T20:35:47,347 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/WALs/jenkins-hbase3.apache.org,46803,1694032527898/jenkins-hbase3.apache.org%2C46803%2C1694032527898.meta.1694032530884.meta with entries=36, filesize=11.13 KB; new WAL /user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/WALs/jenkins-hbase3.apache.org,46803,1694032527898/jenkins-hbase3.apache.org%2C46803%2C1694032527898.meta.1694032547323.meta 2023-09-06T20:35:47,348 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36635,DS-c9c70a6a-ab73-4861-a2f6-0e02c09a2252,DISK]] 2023-09-06T20:35:47,348 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/WALs/jenkins-hbase3.apache.org,46803,1694032527898/jenkins-hbase3.apache.org%2C46803%2C1694032527898.meta.1694032530884.meta is not closed yet, will try archiving it next time 2023-09-06T20:35:47,348 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C46803%2C1694032527898:(num 1694032530354) roll requested 2023-09-06T20:35:47,360 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/WALs/jenkins-hbase3.apache.org,46803,1694032527898/jenkins-hbase3.apache.org%2C46803%2C1694032527898.1694032547348, exclude list is [], retry=0 2023-09-06T20:35:47,364 DEBUG [rs(jenkins-hbase3.apache.org,46803,1694032527898)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 42 2023-09-06T20:35:47,364 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36635,DS-c9c70a6a-ab73-4861-a2f6-0e02c09a2252,DISK] 2023-09-06T20:35:47,364 INFO [rs(jenkins-hbase3.apache.org,46803,1694032527898)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1694032547348 on jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:47,364 DEBUG [rs(jenkins-hbase3.apache.org,46803,1694032527898)-backup-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:35:47,366 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:49110, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-09-06T20:35:47,368 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/WALs/jenkins-hbase3.apache.org,46803,1694032527898/jenkins-hbase3.apache.org%2C46803%2C1694032527898.1694032530354 with entries=19, filesize=3.99 KB; new WAL /user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/WALs/jenkins-hbase3.apache.org,46803,1694032527898/jenkins-hbase3.apache.org%2C46803%2C1694032527898.1694032547348 2023-09-06T20:35:47,368 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36635,DS-c9c70a6a-ab73-4861-a2f6-0e02c09a2252,DISK]] 2023-09-06T20:35:47,368 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/WALs/jenkins-hbase3.apache.org,46803,1694032527898/jenkins-hbase3.apache.org%2C46803%2C1694032527898.1694032530354 is not closed yet, will try archiving it next time 2023-09-06T20:35:47,391 DEBUG [member: 'jenkins-hbase3.apache.org,46803,1694032527898' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-09-06T20:35:47,391 DEBUG [member: 'jenkins-hbase3.apache.org,46803,1694032527898' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,46803,1694032527898' in zk 2023-09-06T20:35:47,394 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:47,394 DEBUG [member: 'jenkins-hbase3.apache.org,46803,1694032527898' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-09-06T20:35:47,394 DEBUG [member: 'jenkins-hbase3.apache.org,46803,1694032527898' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:35:47,394 DEBUG [member: 'jenkins-hbase3.apache.org,46803,1694032527898' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-09-06T20:35:47,394 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:47,395 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:35:47,395 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:35:47,395 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:35:47,396 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:35:47,396 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:35:47,397 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:47,397 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:35:47,398 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:35:47,398 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:47,399 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,46803,1694032527898': 2023-09-06T20:35:47,399 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,46803,1694032527898' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-09-06T20:35:47,399 INFO [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-09-06T20:35:47,400 DEBUG [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-09-06T20:35:47,400 DEBUG [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-09-06T20:35:47,400 DEBUG [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-09-06T20:35:47,400 INFO [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-09-06T20:35:47,402 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:46803-0x100eefe52de0001, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-09-06T20:35:47,402 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-09-06T20:35:47,402 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:46803-0x100eefe52de0001, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-09-06T20:35:47,402 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-09-06T20:35:47,402 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-09-06T20:35:47,402 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:35:47,403 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:35:47,403 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:35:47,403 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:35:47,403 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:35:47,403 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:35:47,404 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:35:47,404 DEBUG [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:47,404 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:35:47,404 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:35:47,405 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:35:47,405 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:47,405 DEBUG [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:47,406 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:35:47,406 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:35:47,407 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:47,415 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:47,415 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-09-06T20:35:47,415 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:46803-0x100eefe52de0001, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-09-06T20:35:47,415 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-09-06T20:35:47,415 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:47,415 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:46803-0x100eefe52de0001, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-09-06T20:35:47,415 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-09-06T20:35:47,415 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:35:47,415 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-09-06T20:35:47,415 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:35:47,415 DEBUG [Listener at localhost/32815-EventThread {}] zookeeper.ZKWatcher(604): master:42111-0x100eefe52de0000, quorum=127.0.0.1:58778, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-09-06T20:35:47,415 DEBUG [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:35:47,415 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-09-06T20:35:47,416 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-09-06T20:35:47,416 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:35:47,416 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:35:47,418 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] client.HBaseAdmin(2838): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-09-06T20:35:47,418 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] client.HBaseAdmin(2846): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-09-06T20:35:47,518 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] client.HBaseAdmin(2852): Getting current status of procedure from master... 2023-09-06T20:35:47,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-09-06T20:35:47,529 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1694032547528_default_test-1694032537394 table=test-1694032537394 type=FLUSH ttl=0 } 2023-09-06T20:35:47,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032547529 (current time:1694032547529). 2023-09-06T20:35:47,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:35:47,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1694032547528_default_test-1694032537394 VERSION not specified, setting to 2 2023-09-06T20:35:47,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x65a53b00 to 127.0.0.1:58778 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:35:47,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@746e5c3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:35:47,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:35:47,544 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:37670, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:35:47,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x65a53b00 to 127.0.0.1:58778 2023-09-06T20:35:47,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:35:47,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:35:47,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] procedure2.ProcedureExecutor(1032): Stored pid=34, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1694032547528_default_test-1694032537394 table=test-1694032537394 type=FLUSH ttl=0 } 2023-09-06T20:35:47,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1694032547528_default_test-1694032537394 table=test-1694032537394 type=FLUSH ttl=0 }, snapshot procedure id = 34 2023-09-06T20:35:47,549 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1694032547528_default_test-1694032537394 table=test-1694032537394 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:35:47,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-09-06T20:35:47,550 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1694032547528_default_test-1694032537394 table=test-1694032537394 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:35:47,552 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1694032547528_default_test-1694032537394 table=test-1694032537394 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:35:47,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-09-06T20:35:47,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-09-06T20:35:47,962 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1694032547528_default_test-1694032537394 table=test-1694032537394 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:35:47,962 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure 7595eee2e2ac3713edf2a8ba161cb7f5}] 2023-09-06T20:35:47,964 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure 7595eee2e2ac3713edf2a8ba161cb7f5 2023-09-06T20:35:48,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46803 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=35 2023-09-06T20:35:48,117 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1694032537394,,1694032537883.7595eee2e2ac3713edf2a8ba161cb7f5. 2023-09-06T20:35:48,118 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2819): Flushing 7595eee2e2ac3713edf2a8ba161cb7f5 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-09-06T20:35:48,136 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/default/test-1694032537394/7595eee2e2ac3713edf2a8ba161cb7f5/.tmp/f/6479ee77687c48c89ac9d7a202cbce01 is 37, key is row10/f:q1/1694032538539/Put/seqid=0 2023-09-06T20:35:48,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-09-06T20:35:48,546 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/default/test-1694032537394/7595eee2e2ac3713edf2a8ba161cb7f5/.tmp/f/6479ee77687c48c89ac9d7a202cbce01 2023-09-06T20:35:48,557 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/default/test-1694032537394/7595eee2e2ac3713edf2a8ba161cb7f5/.tmp/f/6479ee77687c48c89ac9d7a202cbce01 as hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/default/test-1694032537394/7595eee2e2ac3713edf2a8ba161cb7f5/f/6479ee77687c48c89ac9d7a202cbce01 2023-09-06T20:35:48,568 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/default/test-1694032537394/7595eee2e2ac3713edf2a8ba161cb7f5/f/6479ee77687c48c89ac9d7a202cbce01, entries=99, sequenceid=103, filesize=8.2 K 2023-09-06T20:35:48,570 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 7595eee2e2ac3713edf2a8ba161cb7f5 in 452ms, sequenceid=103, compaction requested=false 2023-09-06T20:35:48,571 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2520): Flush status journal for 7595eee2e2ac3713edf2a8ba161cb7f5: 2023-09-06T20:35:48,571 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1694032537394,,1694032537883.7595eee2e2ac3713edf2a8ba161cb7f5. for snapshot_1694032547528_default_test-1694032537394 completed. 2023-09-06T20:35:48,571 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(240): Storing 'test-1694032537394,,1694032537883.7595eee2e2ac3713edf2a8ba161cb7f5.' region-info for snapshot=snapshot_1694032547528_default_test-1694032537394 2023-09-06T20:35:48,571 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:35:48,571 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/default/test-1694032537394/7595eee2e2ac3713edf2a8ba161cb7f5/f/6479ee77687c48c89ac9d7a202cbce01] hfiles 2023-09-06T20:35:48,571 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/default/test-1694032537394/7595eee2e2ac3713edf2a8ba161cb7f5/f/6479ee77687c48c89ac9d7a202cbce01 for snapshot=snapshot_1694032547528_default_test-1694032537394 2023-09-06T20:35:48,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-09-06T20:35:48,983 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1694032537394,,1694032537883.7595eee2e2ac3713edf2a8ba161cb7f5. 2023-09-06T20:35:48,983 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=35 2023-09-06T20:35:48,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.HMaster(3995): Remote procedure done, pid=35 2023-09-06T20:35:48,983 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1694032547528_default_test-1694032537394 on region 7595eee2e2ac3713edf2a8ba161cb7f5 2023-09-06T20:35:48,985 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure 7595eee2e2ac3713edf2a8ba161cb7f5 2023-09-06T20:35:48,988 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=35, resume processing ppid=34 2023-09-06T20:35:48,988 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1694032547528_default_test-1694032537394 table=test-1694032537394 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:35:48,988 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=35, ppid=34, state=SUCCESS; SnapshotRegionProcedure 7595eee2e2ac3713edf2a8ba161cb7f5 in 1.0240 sec 2023-09-06T20:35:48,989 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1694032547528_default_test-1694032537394 table=test-1694032537394 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:35:48,990 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1694032547528_default_test-1694032537394 table=test-1694032537394 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:35:48,990 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1694032547528_default_test-1694032537394 2023-09-06T20:35:48,991 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/.hbase-snapshot/.tmp/snapshot_1694032547528_default_test-1694032537394 2023-09-06T20:35:49,064 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-09-06T20:35:49,116 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system_bulk' 2023-09-06T20:35:49,409 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1694032547528_default_test-1694032537394 table=test-1694032537394 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:35:49,416 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1694032547528_default_test-1694032537394 table=test-1694032537394 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:35:49,417 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/.hbase-snapshot/.tmp/snapshot_1694032547528_default_test-1694032537394 to hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/.hbase-snapshot/snapshot_1694032547528_default_test-1694032537394 2023-09-06T20:35:49,418 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1694032547528_default_test-1694032537394 table=test-1694032537394 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:35:49,418 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1694032547528_default_test-1694032537394 table=test-1694032537394 type=FLUSH ttl=0 }, snapshot procedure id = 34 2023-09-06T20:35:49,420 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=34, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1694032547528_default_test-1694032537394 table=test-1694032537394 type=FLUSH ttl=0 } in 1.8720 sec 2023-09-06T20:35:49,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-09-06T20:35:49,657 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] client.HBaseAdmin$TableFuture(3677): Operation: SNAPSHOT, Table Name: default:test-1694032537394, procId: 34 completed 2023-09-06T20:35:49,657 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1694032542641 2023-09-06T20:35:49,657 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-09-06T20:35:49,659 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-09-06T20:35:49,661 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1694032542641_test-1694032537394 2023-09-06T20:35:49,661 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1694032547528_default_test-1694032537394 to hdfs://localhost:42649/backupUT/backup_1694032542641/default/test-1694032537394/ 2023-09-06T20:35:49,671 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-09-06T20:35:49,702 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:42649, inputRoot=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0 2023-09-06T20:35:49,702 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,42111,1694032524831_428231056_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:42649/backupUT/backup_1694032542641/default/test-1694032537394, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:42649/backupUT/backup_1694032542641/default/test-1694032537394/.hbase-snapshot/.tmp/snapshot_1694032547528_default_test-1694032537394 2023-09-06T20:35:49,702 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:42649, inputRoot=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0, snapshotDir=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/.hbase-snapshot/snapshot_1694032547528_default_test-1694032537394. 2023-09-06T20:35:49,715 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/.hbase-snapshot/snapshot_1694032547528_default_test-1694032537394 to hdfs://localhost:42649/backupUT/backup_1694032542641/default/test-1694032537394/.hbase-snapshot/.tmp/snapshot_1694032547528_default_test-1694032537394 2023-09-06T20:35:50,156 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:50,157 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:50,157 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:50,158 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:51,180 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/hadoop-4520001131570229186.jar 2023-09-06T20:35:51,181 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:51,181 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:51,182 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:51,182 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:51,183 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:51,183 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:51,184 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:51,184 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:51,184 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:51,185 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:35:51,186 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-09-06T20:35:51,186 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-09-06T20:35:51,187 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-09-06T20:35:51,188 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-09-06T20:35:51,188 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-09-06T20:35:51,188 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-09-06T20:35:51,189 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-09-06T20:35:51,189 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-09-06T20:35:51,190 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-09-06T20:35:51,190 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-09-06T20:35:51,191 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-09-06T20:35:51,195 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:35:51,195 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:35:51,196 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:35:51,196 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:35:51,196 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:35:51,197 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:35:51,197 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:35:51,256 WARN [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-09-06T20:35:51,278 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1694032547528_default_test-1694032537394' hfile list 2023-09-06T20:35:51,288 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-09-06T20:35:51,536 WARN [Listener at jenkins-hbase3.apache.org/42325 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/mapreduce.cluster.local.dir/1694032551486/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-09-06T20:35:51,537 WARN [Listener at jenkins-hbase3.apache.org/42325 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/mapreduce.cluster.local.dir/1694032551486/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-09-06T20:35:51,696 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-09-06T20:35:51,726 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/default/test-1694032537394/7595eee2e2ac3713edf2a8ba161cb7f5/f/6479ee77687c48c89ac9d7a202cbce01 output=hdfs://localhost:42649/backupUT/backup_1694032542641/default/test-1694032537394/archive/data/default/test-1694032537394/7595eee2e2ac3713edf2a8ba161cb7f5/f/6479ee77687c48c89ac9d7a202cbce01 2023-09-06T20:35:51,726 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 4.004M/sec 2023-09-06T20:35:52,605 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-09-06T20:35:52,606 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-09-06T20:35:52,614 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1694032547528_default_test-1694032537394 2023-09-06T20:35:52,615 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1694032547528_default_test-1694032537394 finished. 2023-09-06T20:35:52,615 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupSystemTable(1014): test-1694032537394 2023-09-06T20:35:52,688 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1694032542641 2023-09-06T20:35:52,688 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-09-06T20:35:53,108 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:42649/backupUT/backup_1694032542641/.backup.manifest 2023-09-06T20:35:53,108 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.TableBackupClient(358): Backup backup_1694032542641 finished: type=FULL,tablelist=test-1694032537394,targetRootDir=hdfs://localhost:42649/backupUT,startts=1694032547263,completets=1694032552685,bytescopied=0 2023-09-06T20:35:53,108 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-09-06T20:35:53,109 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1694032547528_default_test-1694032537394 2023-09-06T20:35:53,119 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1694032547528_default_test-1694032537394" 2023-09-06T20:35:53,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1694032547528_default_test-1694032537394 2023-09-06T20:35:53,125 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1694032547528_default_test-1694032537394 for backup backup_1694032542641 succeeded. 2023-09-06T20:35:53,126 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-09-06T20:35:53,138 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-09-06T20:35:53,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-09-06T20:35:53,142 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-09-06T20:35:53,150 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-09-06T20:35:53,169 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] impl.TableBackupClient(376): Backup backup_1694032542641 completed. 2023-09-06T20:35:53,169 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-09-06T20:35:53,170 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x34ced1ef to 127.0.0.1:58778 2023-09-06T20:35:53,170 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:35:53,193 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] backup.TestBackupDelete(90): backup complete 2023-09-06T20:35:53,218 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33be3054, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:35:53,219 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:35:53,222 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:45862, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:35:53,233 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:35:53,236 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:47394, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:35:53,274 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-09-06T20:35:53,285 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:35:53,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032553285 (current time:1694032553285). 2023-09-06T20:35:53,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:35:53,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-09-06T20:35:53,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@523599bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:35:53,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:35:53,301 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:47396, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:35:53,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:35:53,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:35:53,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] procedure2.ProcedureExecutor(1032): Stored pid=36, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:35:53,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 36 2023-09-06T20:35:53,306 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:35:53,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-09-06T20:35:53,307 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:35:53,310 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:35:53,321 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:35:53,321 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE; SnapshotRegionProcedure c972ac6cf58538c05726e9a461c31a7f}] 2023-09-06T20:35:53,323 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=37, ppid=36, state=RUNNABLE; SnapshotRegionProcedure c972ac6cf58538c05726e9a461c31a7f 2023-09-06T20:35:53,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-09-06T20:35:53,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46803 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=37 2023-09-06T20:35:53,475 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f. 2023-09-06T20:35:53,476 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2819): Flushing c972ac6cf58538c05726e9a461c31a7f 2/2 column families, dataSize=1.14 KB heapSize=2.37 KB 2023-09-06T20:35:53,491 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/.tmp/meta/87d67d6dd77c4ec7aaac3e7f21bf4489 is 169, key is trslm:hdfs://localhost:42649/backupUT\x00test-1694032537394/meta:log-roll-map/1694032552675/Put/seqid=0 2023-09-06T20:35:53,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-09-06T20:35:53,901 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=586 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/.tmp/meta/87d67d6dd77c4ec7aaac3e7f21bf4489 2023-09-06T20:35:53,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-09-06T20:35:53,914 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/.tmp/session/da3d2282a6d64394ab84e6d230831f40 is 310, key is session:backup_1694032542641/session:context/1694032553148/Put/seqid=0 2023-09-06T20:35:54,326 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=579 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/.tmp/session/da3d2282a6d64394ab84e6d230831f40 2023-09-06T20:35:54,337 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/.tmp/meta/87d67d6dd77c4ec7aaac3e7f21bf4489 as hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/87d67d6dd77c4ec7aaac3e7f21bf4489 2023-09-06T20:35:54,343 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/87d67d6dd77c4ec7aaac3e7f21bf4489, entries=5, sequenceid=18, filesize=5.5 K 2023-09-06T20:35:54,344 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/.tmp/session/da3d2282a6d64394ab84e6d230831f40 as hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/da3d2282a6d64394ab84e6d230831f40 2023-09-06T20:35:54,351 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/da3d2282a6d64394ab84e6d230831f40, entries=2, sequenceid=18, filesize=5.3 K 2023-09-06T20:35:54,352 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1165, heapSize ~2.34 KB/2392, currentSize=0 B/0 for c972ac6cf58538c05726e9a461c31a7f in 877ms, sequenceid=18, compaction requested=false 2023-09-06T20:35:54,352 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2520): Flush status journal for c972ac6cf58538c05726e9a461c31a7f: 2023-09-06T20:35:54,352 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f. for snapshot_backup_system completed. 2023-09-06T20:35:54,352 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f.' region-info for snapshot=snapshot_backup_system 2023-09-06T20:35:54,352 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:35:54,352 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/87d67d6dd77c4ec7aaac3e7f21bf4489] hfiles 2023-09-06T20:35:54,352 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/87d67d6dd77c4ec7aaac3e7f21bf4489 for snapshot=snapshot_backup_system 2023-09-06T20:35:54,352 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/020ff1bc22044c989574f67c5d443447, hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/da3d2282a6d64394ab84e6d230831f40] hfiles 2023-09-06T20:35:54,352 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/020ff1bc22044c989574f67c5d443447 for snapshot=snapshot_backup_system 2023-09-06T20:35:54,352 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/da3d2282a6d64394ab84e6d230831f40 for snapshot=snapshot_backup_system 2023-09-06T20:35:54,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-09-06T20:35:54,764 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f. 2023-09-06T20:35:54,764 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=37 2023-09-06T20:35:54,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.HMaster(3995): Remote procedure done, pid=37 2023-09-06T20:35:54,765 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region c972ac6cf58538c05726e9a461c31a7f 2023-09-06T20:35:54,766 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=37, ppid=36, state=RUNNABLE; SnapshotRegionProcedure c972ac6cf58538c05726e9a461c31a7f 2023-09-06T20:35:54,769 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=37, resume processing ppid=36 2023-09-06T20:35:54,769 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:35:54,769 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=37, ppid=36, state=SUCCESS; SnapshotRegionProcedure c972ac6cf58538c05726e9a461c31a7f in 1.4460 sec 2023-09-06T20:35:54,770 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:35:54,771 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:35:54,771 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-09-06T20:35:54,772 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/.hbase-snapshot/.tmp/snapshot_backup_system 2023-09-06T20:35:55,183 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:35:55,193 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:35:55,194 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/.hbase-snapshot/snapshot_backup_system 2023-09-06T20:35:55,195 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:35:55,196 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 36 2023-09-06T20:35:55,197 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=36, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.8930 sec 2023-09-06T20:35:55,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-09-06T20:35:55,423 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupAdminImpl(227): Deleting backup backup_1694032542641 ... 2023-09-06T20:35:55,426 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] util.BackupUtils(435): Delete log files: .backup.manifest 2023-09-06T20:35:55,427 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] util.BackupUtils(435): Delete log files: default 2023-09-06T20:35:55,427 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] util.BackupUtils(443): Trying to cleanup up target dir : backup_1694032542641 2023-09-06T20:35:55,428 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] util.BackupUtils(458): No data has been found in hdfs://localhost:42649/backupUT/backup_1694032542641/default/test-1694032537394. 2023-09-06T20:35:55,429 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] util.BackupUtils(465): hdfs://localhost:42649/backupUT/backup_1694032542641/default is empty, remove it. 2023-09-06T20:35:55,438 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupAdminImpl(269): 0 bulk loaded files out of 0 were deleted 2023-09-06T20:35:55,439 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupSystemTable(433): deleted 0 original bulkload rows 2023-09-06T20:35:55,443 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupAdminImpl(276): Delete backup backup_1694032542641 completed. 2023-09-06T20:35:55,451 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupSystemTable(1014): test-1694032537394 2023-09-06T20:35:55,454 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-09-06T20:35:55,461 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-09-06T20:35:55,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-09-06T20:35:55,465 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-09-06T20:35:55,465 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-09-06T20:35:55,469 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:35:55,469 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] backup.TestBackupDelete(103): delete_backup 2023-09-06T20:35:55,469 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] backup.TestBackupDelete(105): Deleted 1 backups. Total requested: 1 2023-09-06T20:35:55,493 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] hbase.ResourceChecker(175): after: backup.TestBackupDelete#testBackupDeleteCommand Thread=598 (was 584) Potentially hanging thread: member: 'jenkins-hbase3.apache.org,46803,1694032527898' subprocedure-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-259322142-172.31.12.81-1694032521686:blk_1073741855_1031, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2091846099_16 at /127.0.0.1:50408 [Waiting for operation #16] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x12ffd531-metaLookup-shared--pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1779445525_16 at /127.0.0.1:35064 [Waiting for operation #10] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2091846099_16 at /127.0.0.1:50434 [Waiting for operation #12] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0xc2c31e7-shared-pool-6 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:58778@0x6a321884-SendThread(127.0.0.1:58778) java.lang.Thread.sleep(Native Method) org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:228) org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1338) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1261) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:58778@0x6a321884-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:58778@0x6a321884 java.lang.Object.wait(Native Method) java.lang.Thread.join(Thread.java:1257) java.lang.Thread.join(Thread.java:1331) org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1455) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1485) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1415) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$482/520385432.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x12ffd531-shared-pool-4 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0xc2c31e7-shared-pool-8 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,46803,1694032527898)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1597621837_16 at /127.0.0.1:50368 [Receiving block BP-259322142-172.31.12.81-1694032521686:blk_1073741855_1031] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0xc2c31e7-shared-pool-7 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/cluster_4bbcbad3-1dd9-986e-e95b-c3cf1f3afe53/dfs/data/data1/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1597621837_16 at /127.0.0.1:50378 [Receiving block BP-259322142-172.31.12.81-1694032521686:blk_1073741856_1032] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: (jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x12ffd531-shared-pool-3 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-259322142-172.31.12.81-1694032521686:blk_1073741856_1032, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=682 (was 660) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=129 (was 139), ProcessCount=167 (was 167), AvailableMemoryMB=3263 (was 3459) 2023-09-06T20:35:55,493 WARN [Listener at jenkins-hbase3.apache.org/42325 {}] hbase.ResourceChecker(130): Thread=598 is superior to 500 2023-09-06T20:35:55,513 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] hbase.ResourceChecker(147): before: backup.TestBackupDelete#testBackupDelete Thread=598, OpenFileDescriptor=682, MaxFileDescriptor=60000, SystemLoadAverage=129, ProcessCount=167, AvailableMemoryMB=3261 2023-09-06T20:35:55,513 WARN [Listener at jenkins-hbase3.apache.org/42325 {}] hbase.ResourceChecker(130): Thread=598 is superior to 500 2023-09-06T20:35:55,513 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] backup.TestBackupDelete(59): test backup delete on a single table with data 2023-09-06T20:35:55,523 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f6f1a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:35:55,527 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:35:55,530 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:47402, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:35:55,532 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:35:55,533 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:45868, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:35:55,551 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-09-06T20:35:55,561 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:35:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032555561 (current time:1694032555561). 2023-09-06T20:35:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:35:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-09-06T20:35:55,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5822a24a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:35:55,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:35:55,575 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:47412, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:35:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:35:55,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:35:55,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] procedure2.ProcedureExecutor(1032): Stored pid=38, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:35:55,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 38 2023-09-06T20:35:55,580 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:35:55,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=38 2023-09-06T20:35:55,581 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:35:55,584 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:35:55,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=38 2023-09-06T20:35:55,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=38 2023-09-06T20:35:55,998 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:35:55,998 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE; SnapshotRegionProcedure c972ac6cf58538c05726e9a461c31a7f}] 2023-09-06T20:35:56,000 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=39, ppid=38, state=RUNNABLE; SnapshotRegionProcedure c972ac6cf58538c05726e9a461c31a7f 2023-09-06T20:35:56,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46803 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=39 2023-09-06T20:35:56,154 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f. 2023-09-06T20:35:56,154 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.HRegion(2819): Flushing c972ac6cf58538c05726e9a461c31a7f 2/2 column families, dataSize=268 B heapSize=1.13 KB 2023-09-06T20:35:56,164 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/.tmp/meta/6f8043cc079d486eb37596a5145dd799 is 91, key is incrbackupset:hdfs://localhost:42649/backupUT/meta:test-1694032537394/1694032555452/Put/seqid=0 2023-09-06T20:35:56,177 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=124 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/.tmp/meta/6f8043cc079d486eb37596a5145dd799 2023-09-06T20:35:56,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=38 2023-09-06T20:35:56,185 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6f8043cc079d486eb37596a5145dd799 2023-09-06T20:35:56,190 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/.tmp/session/63d2131a83ba4874a6a538c8a070219b is 59, key is session:backup_1694032542641/session:/1694032555441/DeleteFamily/seqid=0 2023-09-06T20:35:56,195 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=144 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/.tmp/session/63d2131a83ba4874a6a538c8a070219b 2023-09-06T20:35:56,205 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 63d2131a83ba4874a6a538c8a070219b 2023-09-06T20:35:56,206 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/.tmp/meta/6f8043cc079d486eb37596a5145dd799 as hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/6f8043cc079d486eb37596a5145dd799 2023-09-06T20:35:56,213 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6f8043cc079d486eb37596a5145dd799 2023-09-06T20:35:56,213 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/6f8043cc079d486eb37596a5145dd799, entries=2, sequenceid=26, filesize=5.2 K 2023-09-06T20:35:56,215 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/.tmp/session/63d2131a83ba4874a6a538c8a070219b as hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/63d2131a83ba4874a6a538c8a070219b 2023-09-06T20:35:56,225 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 63d2131a83ba4874a6a538c8a070219b 2023-09-06T20:35:56,225 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/63d2131a83ba4874a6a538c8a070219b, entries=2, sequenceid=26, filesize=5.2 K 2023-09-06T20:35:56,226 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.HRegion(3022): Finished flush of dataSize ~268 B/268, heapSize ~1.09 KB/1120, currentSize=0 B/0 for c972ac6cf58538c05726e9a461c31a7f in 72ms, sequenceid=26, compaction requested=true 2023-09-06T20:35:56,226 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.HRegion(2520): Flush status journal for c972ac6cf58538c05726e9a461c31a7f: 2023-09-06T20:35:56,226 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f. for snapshot_backup_system completed. 2023-09-06T20:35:56,226 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f.' region-info for snapshot=snapshot_backup_system 2023-09-06T20:35:56,226 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:35:56,226 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/87d67d6dd77c4ec7aaac3e7f21bf4489, hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/6f8043cc079d486eb37596a5145dd799] hfiles 2023-09-06T20:35:56,226 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/87d67d6dd77c4ec7aaac3e7f21bf4489 for snapshot=snapshot_backup_system 2023-09-06T20:35:56,226 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/6f8043cc079d486eb37596a5145dd799 for snapshot=snapshot_backup_system 2023-09-06T20:35:56,226 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/020ff1bc22044c989574f67c5d443447, hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/da3d2282a6d64394ab84e6d230831f40, hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/63d2131a83ba4874a6a538c8a070219b] hfiles 2023-09-06T20:35:56,226 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/020ff1bc22044c989574f67c5d443447 for snapshot=snapshot_backup_system 2023-09-06T20:35:56,227 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/da3d2282a6d64394ab84e6d230831f40 for snapshot=snapshot_backup_system 2023-09-06T20:35:56,227 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/63d2131a83ba4874a6a538c8a070219b for snapshot=snapshot_backup_system 2023-09-06T20:35:56,633 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f. 2023-09-06T20:35:56,634 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=39 2023-09-06T20:35:56,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.HMaster(3995): Remote procedure done, pid=39 2023-09-06T20:35:56,634 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region c972ac6cf58538c05726e9a461c31a7f 2023-09-06T20:35:56,635 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=39, ppid=38, state=RUNNABLE; SnapshotRegionProcedure c972ac6cf58538c05726e9a461c31a7f 2023-09-06T20:35:56,638 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=39, resume processing ppid=38 2023-09-06T20:35:56,638 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=39, ppid=38, state=SUCCESS; SnapshotRegionProcedure c972ac6cf58538c05726e9a461c31a7f in 638 msec 2023-09-06T20:35:56,638 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:35:56,639 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:35:56,640 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:35:56,640 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-09-06T20:35:56,641 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/.hbase-snapshot/.tmp/snapshot_backup_system 2023-09-06T20:35:56,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=38 2023-09-06T20:35:57,055 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:35:57,065 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:35:57,066 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/.hbase-snapshot/snapshot_backup_system 2023-09-06T20:35:57,067 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:35:57,067 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 38 2023-09-06T20:35:57,069 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=38, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.4900 sec 2023-09-06T20:35:57,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=38 2023-09-06T20:35:57,688 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] impl.TableBackupClient(115): Backup backup_1694032555523 started at 1694032557688. 2023-09-06T20:35:57,690 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.TableBackupClient(119): Backup session backup_1694032555523 has been started. 2023-09-06T20:35:57,692 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-09-06T20:35:57,693 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-09-06T20:35:57,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] procedure.ProcedureCoordinator(143): Procedure rolllog was in running list but was completed. Accepting new attempt. 2023-09-06T20:35:57,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-09-06T20:35:57,693 INFO [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-09-06T20:35:57,693 DEBUG [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-09-06T20:35:57,694 DEBUG [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-09-06T20:35:57,695 DEBUG [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-09-06T20:35:57,695 DEBUG [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:35:57,697 DEBUG [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:57,697 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:35:57,697 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:35:57,697 DEBUG [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-09-06T20:35:57,697 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-09-06T20:35:57,698 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-09-06T20:35:57,698 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:35:57,698 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-09-06T20:35:57,698 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-09-06T20:35:57,701 WARN [zk-event-processor-pool-0 {}] procedure.ProcedureMember(133): A completed old subproc rolllog is still present, removing 2023-09-06T20:35:57,701 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-09-06T20:35:57,701 DEBUG [member: 'jenkins-hbase3.apache.org,46803,1694032527898' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-09-06T20:35:57,701 DEBUG [member: 'jenkins-hbase3.apache.org,46803,1694032527898' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-09-06T20:35:57,702 DEBUG [member: 'jenkins-hbase3.apache.org,46803,1694032527898' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-09-06T20:35:57,702 DEBUG [member: 'jenkins-hbase3.apache.org,46803,1694032527898' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-09-06T20:35:57,702 DEBUG [member: 'jenkins-hbase3.apache.org,46803,1694032527898' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,46803,1694032527898' joining acquired barrier for procedure (rolllog) in zk 2023-09-06T20:35:57,706 DEBUG [member: 'jenkins-hbase3.apache.org,46803,1694032527898' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-09-06T20:35:57,706 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:57,706 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:35:57,706 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:35:57,706 DEBUG [member: 'jenkins-hbase3.apache.org,46803,1694032527898' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-09-06T20:35:57,706 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:35:57,707 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:35:57,707 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:35:57,707 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:57,708 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:35:57,708 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,46803,1694032527898' joining acquired barrier for procedure 'rolllog' on coordinator 2023-09-06T20:35:57,708 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@2401bfc6[Count = 0] remaining members to acquire global barrier 2023-09-06T20:35:57,708 DEBUG [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-09-06T20:35:57,708 DEBUG [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-09-06T20:35:57,710 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-09-06T20:35:57,710 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-09-06T20:35:57,710 DEBUG [member: 'jenkins-hbase3.apache.org,46803,1694032527898' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-09-06T20:35:57,710 DEBUG [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-09-06T20:35:57,710 DEBUG [member: 'jenkins-hbase3.apache.org,46803,1694032527898' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-09-06T20:35:57,710 DEBUG [rs(jenkins-hbase3.apache.org,46803,1694032527898)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:57,710 INFO [rs(jenkins-hbase3.apache.org,46803,1694032527898)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1694032547348 highest: 1694032547348 on jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:57,710 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C46803%2C1694032527898.meta:.meta(num 1694032547323) roll requested 2023-09-06T20:35:57,722 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/WALs/jenkins-hbase3.apache.org,46803,1694032527898/jenkins-hbase3.apache.org%2C46803%2C1694032527898.meta.1694032557711.meta, exclude list is [], retry=0 2023-09-06T20:35:57,726 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36635,DS-c9c70a6a-ab73-4861-a2f6-0e02c09a2252,DISK] 2023-09-06T20:35:57,728 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/WALs/jenkins-hbase3.apache.org,46803,1694032527898/jenkins-hbase3.apache.org%2C46803%2C1694032527898.meta.1694032547323.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/WALs/jenkins-hbase3.apache.org,46803,1694032527898/jenkins-hbase3.apache.org%2C46803%2C1694032527898.meta.1694032557711.meta 2023-09-06T20:35:57,728 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36635,DS-c9c70a6a-ab73-4861-a2f6-0e02c09a2252,DISK]] 2023-09-06T20:35:57,728 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/WALs/jenkins-hbase3.apache.org,46803,1694032527898/jenkins-hbase3.apache.org%2C46803%2C1694032527898.meta.1694032547323.meta is not closed yet, will try archiving it next time 2023-09-06T20:35:57,728 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C46803%2C1694032527898:(num 1694032547348) roll requested 2023-09-06T20:35:57,732 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/WALs/jenkins-hbase3.apache.org,46803,1694032527898/jenkins-hbase3.apache.org%2C46803%2C1694032527898.meta.1694032547323.meta to hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/oldWALs/jenkins-hbase3.apache.org%2C46803%2C1694032527898.meta.1694032547323.meta 2023-09-06T20:35:57,741 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/WALs/jenkins-hbase3.apache.org,46803,1694032527898/jenkins-hbase3.apache.org%2C46803%2C1694032527898.1694032557728, exclude list is [], retry=0 2023-09-06T20:35:57,747 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36635,DS-c9c70a6a-ab73-4861-a2f6-0e02c09a2252,DISK] 2023-09-06T20:35:57,749 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/WALs/jenkins-hbase3.apache.org,46803,1694032527898/jenkins-hbase3.apache.org%2C46803%2C1694032527898.1694032547348 with entries=20, filesize=3.96 KB; new WAL /user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/WALs/jenkins-hbase3.apache.org,46803,1694032527898/jenkins-hbase3.apache.org%2C46803%2C1694032527898.1694032557728 2023-09-06T20:35:57,749 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36635,DS-c9c70a6a-ab73-4861-a2f6-0e02c09a2252,DISK]] 2023-09-06T20:35:57,749 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/WALs/jenkins-hbase3.apache.org,46803,1694032527898/jenkins-hbase3.apache.org%2C46803%2C1694032527898.1694032547348 is not closed yet, will try archiving it next time 2023-09-06T20:35:57,751 DEBUG [rs(jenkins-hbase3.apache.org,46803,1694032527898)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 41 2023-09-06T20:35:57,751 INFO [rs(jenkins-hbase3.apache.org,46803,1694032527898)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1694032557728 on jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:57,753 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/WALs/jenkins-hbase3.apache.org,46803,1694032527898/jenkins-hbase3.apache.org%2C46803%2C1694032527898.1694032547348 not finished, retry = 0 2023-09-06T20:35:57,765 DEBUG [member: 'jenkins-hbase3.apache.org,46803,1694032527898' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-09-06T20:35:57,765 DEBUG [member: 'jenkins-hbase3.apache.org,46803,1694032527898' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,46803,1694032527898' in zk 2023-09-06T20:35:57,768 DEBUG [member: 'jenkins-hbase3.apache.org,46803,1694032527898' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-09-06T20:35:57,768 DEBUG [member: 'jenkins-hbase3.apache.org,46803,1694032527898' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:35:57,768 DEBUG [member: 'jenkins-hbase3.apache.org,46803,1694032527898' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-09-06T20:35:57,769 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:57,769 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:35:57,769 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:35:57,769 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:35:57,770 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:35:57,770 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:35:57,771 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:57,771 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:35:57,772 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:35:57,772 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:57,773 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,46803,1694032527898': 2023-09-06T20:35:57,773 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,46803,1694032527898' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-09-06T20:35:57,773 INFO [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-09-06T20:35:57,773 DEBUG [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-09-06T20:35:57,773 DEBUG [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-09-06T20:35:57,773 DEBUG [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-09-06T20:35:57,773 INFO [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-09-06T20:35:57,775 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-09-06T20:35:57,775 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:35:57,775 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-09-06T20:35:57,775 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:35:57,775 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:35:57,775 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:35:57,775 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:35:57,776 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:35:57,776 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:35:57,776 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:35:57,776 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:35:57,777 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:35:57,777 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:57,777 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:35:57,778 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:35:57,778 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:35:57,785 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:35:57,785 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:35:57,785 DEBUG [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:35:57,785 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-09-06T20:35:57,785 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-09-06T20:35:57,787 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:35:57,787 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:35:57,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-09-06T20:35:57,891 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1694032557890_default_test-1694032537394 table=test-1694032537394 type=FLUSH ttl=0 } 2023-09-06T20:35:57,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032557891 (current time:1694032557891). 2023-09-06T20:35:57,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:35:57,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1694032557890_default_test-1694032537394 VERSION not specified, setting to 2 2023-09-06T20:35:57,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33b6c499, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:35:57,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:35:57,902 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:47426, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:35:57,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:35:57,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:35:57,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] procedure2.ProcedureExecutor(1032): Stored pid=40, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_1694032557890_default_test-1694032537394 table=test-1694032537394 type=FLUSH ttl=0 } 2023-09-06T20:35:57,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1694032557890_default_test-1694032537394 table=test-1694032537394 type=FLUSH ttl=0 }, snapshot procedure id = 40 2023-09-06T20:35:57,907 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_1694032557890_default_test-1694032537394 table=test-1694032537394 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:35:57,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=40 2023-09-06T20:35:57,908 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_1694032557890_default_test-1694032537394 table=test-1694032537394 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:35:57,910 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_1694032557890_default_test-1694032537394 table=test-1694032537394 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:35:58,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=40 2023-09-06T20:35:58,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=40 2023-09-06T20:35:58,318 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_1694032557890_default_test-1694032537394 table=test-1694032537394 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:35:58,318 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE; SnapshotRegionProcedure 7595eee2e2ac3713edf2a8ba161cb7f5}] 2023-09-06T20:35:58,326 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=41, ppid=40, state=RUNNABLE; SnapshotRegionProcedure 7595eee2e2ac3713edf2a8ba161cb7f5 2023-09-06T20:35:58,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46803 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=41 2023-09-06T20:35:58,479 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1694032537394,,1694032537883.7595eee2e2ac3713edf2a8ba161cb7f5. 2023-09-06T20:35:58,479 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.HRegion(2520): Flush status journal for 7595eee2e2ac3713edf2a8ba161cb7f5: 2023-09-06T20:35:58,479 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1694032537394,,1694032537883.7595eee2e2ac3713edf2a8ba161cb7f5. for snapshot_1694032557890_default_test-1694032537394 completed. 2023-09-06T20:35:58,480 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] snapshot.SnapshotManifest(240): Storing 'test-1694032537394,,1694032537883.7595eee2e2ac3713edf2a8ba161cb7f5.' region-info for snapshot=snapshot_1694032557890_default_test-1694032537394 2023-09-06T20:35:58,480 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:35:58,480 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/default/test-1694032537394/7595eee2e2ac3713edf2a8ba161cb7f5/f/6479ee77687c48c89ac9d7a202cbce01] hfiles 2023-09-06T20:35:58,480 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/default/test-1694032537394/7595eee2e2ac3713edf2a8ba161cb7f5/f/6479ee77687c48c89ac9d7a202cbce01 for snapshot=snapshot_1694032557890_default_test-1694032537394 2023-09-06T20:35:58,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=40 2023-09-06T20:35:58,887 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1694032537394,,1694032537883.7595eee2e2ac3713edf2a8ba161cb7f5. 2023-09-06T20:35:58,887 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=41 2023-09-06T20:35:58,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.HMaster(3995): Remote procedure done, pid=41 2023-09-06T20:35:58,887 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1694032557890_default_test-1694032537394 on region 7595eee2e2ac3713edf2a8ba161cb7f5 2023-09-06T20:35:58,889 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=41, ppid=40, state=RUNNABLE; SnapshotRegionProcedure 7595eee2e2ac3713edf2a8ba161cb7f5 2023-09-06T20:35:58,891 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=41, resume processing ppid=40 2023-09-06T20:35:58,891 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_1694032557890_default_test-1694032537394 table=test-1694032537394 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:35:58,892 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=41, ppid=40, state=SUCCESS; SnapshotRegionProcedure 7595eee2e2ac3713edf2a8ba161cb7f5 in 572 msec 2023-09-06T20:35:58,892 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_1694032557890_default_test-1694032537394 table=test-1694032537394 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:35:58,893 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_1694032557890_default_test-1694032537394 table=test-1694032537394 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:35:58,893 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1694032557890_default_test-1694032537394 2023-09-06T20:35:58,894 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/.hbase-snapshot/.tmp/snapshot_1694032557890_default_test-1694032537394 2023-09-06T20:35:59,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=40 2023-09-06T20:35:59,306 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_1694032557890_default_test-1694032537394 table=test-1694032537394 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:35:59,313 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_1694032557890_default_test-1694032537394 table=test-1694032537394 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:35:59,314 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/.hbase-snapshot/.tmp/snapshot_1694032557890_default_test-1694032537394 to hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/.hbase-snapshot/snapshot_1694032557890_default_test-1694032537394 2023-09-06T20:35:59,316 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_1694032557890_default_test-1694032537394 table=test-1694032537394 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:35:59,316 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1694032557890_default_test-1694032537394 table=test-1694032537394 type=FLUSH ttl=0 }, snapshot procedure id = 40 2023-09-06T20:35:59,318 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=40, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_1694032557890_default_test-1694032537394 table=test-1694032537394 type=FLUSH ttl=0 } in 1.4120 sec 2023-09-06T20:36:00,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=40 2023-09-06T20:36:00,016 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1694032555523 2023-09-06T20:36:00,016 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-09-06T20:36:00,016 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-09-06T20:36:00,016 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1694032555523_test-1694032537394 2023-09-06T20:36:00,016 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1694032557890_default_test-1694032537394 to hdfs://localhost:42649/backupUT/backup_1694032555523/default/test-1694032537394/ 2023-09-06T20:36:00,016 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-09-06T20:36:00,041 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:42649, inputRoot=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0 2023-09-06T20:36:00,042 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,42111,1694032524831_-1468073295_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:42649/backupUT/backup_1694032555523/default/test-1694032537394, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:42649/backupUT/backup_1694032555523/default/test-1694032537394/.hbase-snapshot/.tmp/snapshot_1694032557890_default_test-1694032537394 2023-09-06T20:36:00,042 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:42649, inputRoot=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0, snapshotDir=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/.hbase-snapshot/snapshot_1694032557890_default_test-1694032537394. 2023-09-06T20:36:00,052 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/.hbase-snapshot/snapshot_1694032557890_default_test-1694032537394 to hdfs://localhost:42649/backupUT/backup_1694032555523/default/test-1694032537394/.hbase-snapshot/.tmp/snapshot_1694032557890_default_test-1694032537394 2023-09-06T20:36:00,470 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:36:00,470 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-09-06T20:36:00,471 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-09-06T20:36:00,471 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-09-06T20:36:01,575 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/hadoop-2069500251029050914.jar 2023-09-06T20:36:01,576 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:36:01,576 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:36:01,576 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:36:01,577 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-09-06T20:36:01,578 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-09-06T20:36:01,578 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-09-06T20:36:01,579 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-09-06T20:36:01,579 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-09-06T20:36:01,580 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-09-06T20:36:01,581 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:36:01,582 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-09-06T20:36:01,583 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-09-06T20:36:01,583 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-09-06T20:36:01,584 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-09-06T20:36:01,584 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-09-06T20:36:01,585 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-09-06T20:36:01,585 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-09-06T20:36:01,586 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-09-06T20:36:01,586 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-09-06T20:36:01,587 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-09-06T20:36:01,587 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-09-06T20:36:01,588 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:36:01,588 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:36:01,590 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:36:01,590 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:36:01,591 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:36:01,591 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:36:01,592 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:36:01,640 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1694032557890_default_test-1694032537394' hfile list 2023-09-06T20:36:01,643 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-09-06T20:36:01,724 WARN [Listener at jenkins-hbase3.apache.org/42325 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/mapreduce.cluster.local.dir/1694032561686/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-09-06T20:36:01,724 WARN [Listener at jenkins-hbase3.apache.org/42325 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/mapreduce.cluster.local.dir/1694032561686/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-09-06T20:36:01,847 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-09-06T20:36:01,878 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/default/test-1694032537394/7595eee2e2ac3713edf2a8ba161cb7f5/f/6479ee77687c48c89ac9d7a202cbce01 output=hdfs://localhost:42649/backupUT/backup_1694032555523/default/test-1694032537394/archive/data/default/test-1694032537394/7595eee2e2ac3713edf2a8ba161cb7f5/f/6479ee77687c48c89ac9d7a202cbce01 2023-09-06T20:36:01,878 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 4.004M/sec 2023-09-06T20:36:02,794 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-09-06T20:36:02,795 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-09-06T20:36:02,811 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1694032557890_default_test-1694032537394 2023-09-06T20:36:02,811 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1694032557890_default_test-1694032537394 finished. 2023-09-06T20:36:02,811 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupSystemTable(1014): test-1694032537394 2023-09-06T20:36:02,853 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1694032555523 2023-09-06T20:36:02,853 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-09-06T20:36:02,880 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:42649/backupUT/backup_1694032555523/.backup.manifest 2023-09-06T20:36:02,880 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.TableBackupClient(358): Backup backup_1694032555523 finished: type=FULL,tablelist=test-1694032537394,targetRootDir=hdfs://localhost:42649/backupUT,startts=1694032557688,completets=1694032562852,bytescopied=0 2023-09-06T20:36:02,880 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-09-06T20:36:02,881 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1694032557890_default_test-1694032537394 2023-09-06T20:36:02,882 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1694032557890_default_test-1694032537394" 2023-09-06T20:36:02,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1694032557890_default_test-1694032537394 2023-09-06T20:36:02,890 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1694032557890_default_test-1694032537394 for backup backup_1694032555523 succeeded. 2023-09-06T20:36:02,898 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-09-06T20:36:02,906 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-09-06T20:36:02,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-09-06T20:36:02,915 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-09-06T20:36:02,918 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-09-06T20:36:02,925 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] impl.TableBackupClient(376): Backup backup_1694032555523 completed. 2023-09-06T20:36:02,926 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:36:02,946 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] backup.TestBackupDelete(63): backup complete 2023-09-06T20:36:02,979 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-09-06T20:36:02,990 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:36:02,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032562990 (current time:1694032562990). 2023-09-06T20:36:02,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:36:02,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-09-06T20:36:02,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4760debc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:36:03,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:36:03,033 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:38194, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:36:03,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:36:03,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:36:03,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] procedure2.ProcedureExecutor(1032): Stored pid=42, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:36:03,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 42 2023-09-06T20:36:03,039 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:36:03,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-09-06T20:36:03,040 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:36:03,043 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:36:03,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-09-06T20:36:03,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-09-06T20:36:03,449 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:36:03,450 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE; SnapshotRegionProcedure c972ac6cf58538c05726e9a461c31a7f}] 2023-09-06T20:36:03,451 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=43, ppid=42, state=RUNNABLE; SnapshotRegionProcedure c972ac6cf58538c05726e9a461c31a7f 2023-09-06T20:36:03,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46803 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=43 2023-09-06T20:36:03,603 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f. 2023-09-06T20:36:03,604 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegion(2819): Flushing c972ac6cf58538c05726e9a461c31a7f 2/2 column families, dataSize=1.06 KB heapSize=2.22 KB 2023-09-06T20:36:03,609 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/.tmp/meta/00b44b8959014deaaec4aa483bcdf05c is 169, key is trslm:hdfs://localhost:42649/backupUT\x00test-1694032537394/meta:log-roll-map/1694032562822/Put/seqid=0 2023-09-06T20:36:03,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-09-06T20:36:04,015 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=511 B at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/.tmp/meta/00b44b8959014deaaec4aa483bcdf05c 2023-09-06T20:36:04,026 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/.tmp/session/a78b3e89777d43d393d364a9185f21e9 is 310, key is session:backup_1694032555523/session:context/1694032562916/Put/seqid=0 2023-09-06T20:36:04,032 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=579 B at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/.tmp/session/a78b3e89777d43d393d364a9185f21e9 2023-09-06T20:36:04,041 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/.tmp/meta/00b44b8959014deaaec4aa483bcdf05c as hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/00b44b8959014deaaec4aa483bcdf05c 2023-09-06T20:36:04,053 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/00b44b8959014deaaec4aa483bcdf05c, entries=5, sequenceid=38, filesize=5.5 K 2023-09-06T20:36:04,055 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/.tmp/session/a78b3e89777d43d393d364a9185f21e9 as hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/a78b3e89777d43d393d364a9185f21e9 2023-09-06T20:36:04,062 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/a78b3e89777d43d393d364a9185f21e9, entries=2, sequenceid=38, filesize=5.3 K 2023-09-06T20:36:04,063 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegion(3022): Finished flush of dataSize ~1.06 KB/1090, heapSize ~2.19 KB/2240, currentSize=0 B/0 for c972ac6cf58538c05726e9a461c31a7f in 459ms, sequenceid=38, compaction requested=true 2023-09-06T20:36:04,063 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegion(2520): Flush status journal for c972ac6cf58538c05726e9a461c31a7f: 2023-09-06T20:36:04,063 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f. for snapshot_backup_system completed. 2023-09-06T20:36:04,064 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f.' region-info for snapshot=snapshot_backup_system 2023-09-06T20:36:04,064 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:36:04,064 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/87d67d6dd77c4ec7aaac3e7f21bf4489, hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/6f8043cc079d486eb37596a5145dd799, hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/00b44b8959014deaaec4aa483bcdf05c] hfiles 2023-09-06T20:36:04,064 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/87d67d6dd77c4ec7aaac3e7f21bf4489 for snapshot=snapshot_backup_system 2023-09-06T20:36:04,064 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/6f8043cc079d486eb37596a5145dd799 for snapshot=snapshot_backup_system 2023-09-06T20:36:04,064 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/00b44b8959014deaaec4aa483bcdf05c for snapshot=snapshot_backup_system 2023-09-06T20:36:04,064 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/020ff1bc22044c989574f67c5d443447, hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/da3d2282a6d64394ab84e6d230831f40, hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/63d2131a83ba4874a6a538c8a070219b, hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/a78b3e89777d43d393d364a9185f21e9] hfiles 2023-09-06T20:36:04,064 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (1/4): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/020ff1bc22044c989574f67c5d443447 for snapshot=snapshot_backup_system 2023-09-06T20:36:04,064 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (2/4): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/da3d2282a6d64394ab84e6d230831f40 for snapshot=snapshot_backup_system 2023-09-06T20:36:04,064 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (3/4): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/63d2131a83ba4874a6a538c8a070219b for snapshot=snapshot_backup_system 2023-09-06T20:36:04,064 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (4/4): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/a78b3e89777d43d393d364a9185f21e9 for snapshot=snapshot_backup_system 2023-09-06T20:36:04,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-09-06T20:36:04,476 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f. 2023-09-06T20:36:04,477 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=43 2023-09-06T20:36:04,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.HMaster(3995): Remote procedure done, pid=43 2023-09-06T20:36:04,477 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region c972ac6cf58538c05726e9a461c31a7f 2023-09-06T20:36:04,479 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=43, ppid=42, state=RUNNABLE; SnapshotRegionProcedure c972ac6cf58538c05726e9a461c31a7f 2023-09-06T20:36:04,481 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=43, resume processing ppid=42 2023-09-06T20:36:04,481 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:36:04,481 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=43, ppid=42, state=SUCCESS; SnapshotRegionProcedure c972ac6cf58538c05726e9a461c31a7f in 1.0310 sec 2023-09-06T20:36:04,482 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:36:04,483 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:36:04,483 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-09-06T20:36:04,484 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/.hbase-snapshot/.tmp/snapshot_backup_system 2023-09-06T20:36:04,894 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:36:04,904 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:36:04,905 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/.hbase-snapshot/snapshot_backup_system 2023-09-06T20:36:04,906 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-09-06T20:36:04,908 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:36:04,908 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 42 2023-09-06T20:36:04,908 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:57520, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-09-06T20:36:04,909 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=42, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.8710 sec 2023-09-06T20:36:05,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-09-06T20:36:05,154 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupAdminImpl(227): Deleting backup backup_1694032555523 ... 2023-09-06T20:36:05,155 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] util.BackupUtils(435): Delete log files: .backup.manifest 2023-09-06T20:36:05,157 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] util.BackupUtils(435): Delete log files: default 2023-09-06T20:36:05,157 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] util.BackupUtils(443): Trying to cleanup up target dir : backup_1694032555523 2023-09-06T20:36:05,158 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] util.BackupUtils(458): No data has been found in hdfs://localhost:42649/backupUT/backup_1694032555523/default/test-1694032537394. 2023-09-06T20:36:05,159 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] util.BackupUtils(465): hdfs://localhost:42649/backupUT/backup_1694032555523/default is empty, remove it. 2023-09-06T20:36:05,164 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupAdminImpl(269): 0 bulk loaded files out of 0 were deleted 2023-09-06T20:36:05,164 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupSystemTable(433): deleted 0 original bulkload rows 2023-09-06T20:36:05,165 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupAdminImpl(276): Delete backup backup_1694032555523 completed. 2023-09-06T20:36:05,174 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupSystemTable(1014): test-1694032537394 2023-09-06T20:36:05,177 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-09-06T20:36:05,183 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-09-06T20:36:05,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-09-06T20:36:05,186 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-09-06T20:36:05,186 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-09-06T20:36:05,190 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] backup.TestBackupDelete(76): delete_backup 2023-09-06T20:36:05,207 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] hbase.ResourceChecker(175): after: backup.TestBackupDelete#testBackupDelete Thread=601 (was 598) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1597621837_16 at /127.0.0.1:35160 [Receiving block BP-259322142-172.31.12.81-1694032521686:blk_1073741875_1051] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2091846099_16 at /127.0.0.1:35200 [Waiting for operation #12] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x12ffd531-shared-pool-8 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x12ffd531-shared-pool-7 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x12ffd531-shared-pool-6 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/cluster_4bbcbad3-1dd9-986e-e95b-c3cf1f3afe53/dfs/data/data2/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0xc2c31e7-shared-pool-12 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0xc2c31e7-shared-pool-13 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1597621837_16 at /127.0.0.1:35176 [Receiving block BP-259322142-172.31.12.81-1694032521686:blk_1073741876_1052] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x12ffd531-shared-pool-5 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1779445525_16 at /127.0.0.1:39714 [Waiting for operation #12] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x12ffd531-shared-pool-10 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x12ffd531-shared-pool-9 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: WAL-Archive-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2091846099_16 at /127.0.0.1:35138 [Waiting for operation #19] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-259322142-172.31.12.81-1694032521686:blk_1073741876_1052, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-259322142-172.31.12.81-1694032521686:blk_1073741875_1051, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x12ffd531-metaLookup-shared--pool-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0xc2c31e7-shared-pool-11 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=683 (was 682) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=132 (was 129) - SystemLoadAverage LEAK? -, ProcessCount=170 (was 167) - ProcessCount LEAK? -, AvailableMemoryMB=2553 (was 3261) 2023-09-06T20:36:05,207 WARN [Listener at jenkins-hbase3.apache.org/42325 {}] hbase.ResourceChecker(130): Thread=601 is superior to 500 2023-09-06T20:36:05,223 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] hbase.ResourceChecker(147): before: backup.TestBackupDelete#testBackupPurgeOldBackupsCommand Thread=601, OpenFileDescriptor=683, MaxFileDescriptor=60000, SystemLoadAverage=132, ProcessCount=170, AvailableMemoryMB=2553 2023-09-06T20:36:05,223 WARN [Listener at jenkins-hbase3.apache.org/42325 {}] hbase.ResourceChecker(130): Thread=601 is superior to 500 2023-09-06T20:36:05,223 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] backup.TestBackupDelete(111): test backup delete (purge old backups) on a single table with data: command-line 2023-09-06T20:36:05,232 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49276397, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:36:05,235 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:36:05,237 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:38196, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:36:05,239 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:36:05,241 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:57528, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:36:05,260 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-09-06T20:36:05,266 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:36:05,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1693859765266 (current time:1693859765266). 2023-09-06T20:36:05,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:36:05,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-09-06T20:36:05,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@203ec3fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:36:05,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:36:05,281 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:38202, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:36:05,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:36:05,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:36:05,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] procedure2.ProcedureExecutor(1032): Stored pid=44, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:36:05,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 44 2023-09-06T20:36:05,285 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:36:05,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=44 2023-09-06T20:36:05,286 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:36:05,288 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:36:05,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=44 2023-09-06T20:36:05,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=44 2023-09-06T20:36:05,696 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:36:05,696 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE; SnapshotRegionProcedure c972ac6cf58538c05726e9a461c31a7f}] 2023-09-06T20:36:05,698 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=45, ppid=44, state=RUNNABLE; SnapshotRegionProcedure c972ac6cf58538c05726e9a461c31a7f 2023-09-06T20:36:05,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46803 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=45 2023-09-06T20:36:05,850 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f. 2023-09-06T20:36:05,850 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.HRegion(2819): Flushing c972ac6cf58538c05726e9a461c31a7f 2/2 column families, dataSize=268 B heapSize=1.13 KB 2023-09-06T20:36:05,855 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/.tmp/meta/3dee3ea754fe4dcc839a7959c90f1590 is 91, key is incrbackupset:hdfs://localhost:42649/backupUT/meta:test-1694032537394/1694032565175/Put/seqid=0 2023-09-06T20:36:05,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=44 2023-09-06T20:36:06,260 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=124 B at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/.tmp/meta/3dee3ea754fe4dcc839a7959c90f1590 2023-09-06T20:36:06,266 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3dee3ea754fe4dcc839a7959c90f1590 2023-09-06T20:36:06,269 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/.tmp/session/21f35f677df14d5fae225eba13a89b66 is 59, key is session:backup_1694032555523/session:/1694032565165/DeleteFamily/seqid=0 2023-09-06T20:36:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=44 2023-09-06T20:36:06,674 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=144 B at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/.tmp/session/21f35f677df14d5fae225eba13a89b66 2023-09-06T20:36:06,680 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 21f35f677df14d5fae225eba13a89b66 2023-09-06T20:36:06,681 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/.tmp/meta/3dee3ea754fe4dcc839a7959c90f1590 as hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/3dee3ea754fe4dcc839a7959c90f1590 2023-09-06T20:36:06,687 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3dee3ea754fe4dcc839a7959c90f1590 2023-09-06T20:36:06,687 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/3dee3ea754fe4dcc839a7959c90f1590, entries=2, sequenceid=46, filesize=5.2 K 2023-09-06T20:36:06,688 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/.tmp/session/21f35f677df14d5fae225eba13a89b66 as hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/21f35f677df14d5fae225eba13a89b66 2023-09-06T20:36:06,694 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 21f35f677df14d5fae225eba13a89b66 2023-09-06T20:36:06,695 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/21f35f677df14d5fae225eba13a89b66, entries=2, sequenceid=46, filesize=5.2 K 2023-09-06T20:36:06,696 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.HRegion(3022): Finished flush of dataSize ~268 B/268, heapSize ~1.09 KB/1120, currentSize=0 B/0 for c972ac6cf58538c05726e9a461c31a7f in 846ms, sequenceid=46, compaction requested=true 2023-09-06T20:36:06,696 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.HRegion(2520): Flush status journal for c972ac6cf58538c05726e9a461c31a7f: 2023-09-06T20:36:06,696 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f. for snapshot_backup_system completed. 2023-09-06T20:36:06,696 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f.' region-info for snapshot=snapshot_backup_system 2023-09-06T20:36:06,696 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:36:06,696 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/87d67d6dd77c4ec7aaac3e7f21bf4489, hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/6f8043cc079d486eb37596a5145dd799, hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/00b44b8959014deaaec4aa483bcdf05c, hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/3dee3ea754fe4dcc839a7959c90f1590] hfiles 2023-09-06T20:36:06,696 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (1/4): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/87d67d6dd77c4ec7aaac3e7f21bf4489 for snapshot=snapshot_backup_system 2023-09-06T20:36:06,697 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (2/4): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/6f8043cc079d486eb37596a5145dd799 for snapshot=snapshot_backup_system 2023-09-06T20:36:06,697 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (3/4): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/00b44b8959014deaaec4aa483bcdf05c for snapshot=snapshot_backup_system 2023-09-06T20:36:06,697 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (4/4): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/3dee3ea754fe4dcc839a7959c90f1590 for snapshot=snapshot_backup_system 2023-09-06T20:36:06,697 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/020ff1bc22044c989574f67c5d443447, hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/da3d2282a6d64394ab84e6d230831f40, hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/63d2131a83ba4874a6a538c8a070219b, hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/a78b3e89777d43d393d364a9185f21e9, hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/21f35f677df14d5fae225eba13a89b66] hfiles 2023-09-06T20:36:06,697 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (1/5): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/020ff1bc22044c989574f67c5d443447 for snapshot=snapshot_backup_system 2023-09-06T20:36:06,697 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (2/5): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/da3d2282a6d64394ab84e6d230831f40 for snapshot=snapshot_backup_system 2023-09-06T20:36:06,697 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (3/5): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/63d2131a83ba4874a6a538c8a070219b for snapshot=snapshot_backup_system 2023-09-06T20:36:06,697 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (4/5): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/a78b3e89777d43d393d364a9185f21e9 for snapshot=snapshot_backup_system 2023-09-06T20:36:06,697 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (5/5): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/21f35f677df14d5fae225eba13a89b66 for snapshot=snapshot_backup_system 2023-09-06T20:36:07,107 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f. 2023-09-06T20:36:07,107 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=45 2023-09-06T20:36:07,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.HMaster(3995): Remote procedure done, pid=45 2023-09-06T20:36:07,108 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region c972ac6cf58538c05726e9a461c31a7f 2023-09-06T20:36:07,109 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=45, ppid=44, state=RUNNABLE; SnapshotRegionProcedure c972ac6cf58538c05726e9a461c31a7f 2023-09-06T20:36:07,112 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=45, resume processing ppid=44 2023-09-06T20:36:07,112 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:36:07,113 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=45, ppid=44, state=SUCCESS; SnapshotRegionProcedure c972ac6cf58538c05726e9a461c31a7f in 1.4140 sec 2023-09-06T20:36:07,113 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:36:07,114 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:36:07,114 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-09-06T20:36:07,115 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/.hbase-snapshot/.tmp/snapshot_backup_system 2023-09-06T20:36:07,127 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:36:07,143 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:36:07,143 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/.hbase-snapshot/snapshot_backup_system 2023-09-06T20:36:07,145 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:36:07,145 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 44 2023-09-06T20:36:07,149 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=44, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.8620 sec 2023-09-06T20:36:07,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=44 2023-09-06T20:36:07,393 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] impl.TableBackupClient(115): Backup backup_1693859765232 started at 1693859767393. 2023-09-06T20:36:07,396 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.TableBackupClient(119): Backup session backup_1693859765232 has been started. 2023-09-06T20:36:07,397 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-09-06T20:36:07,398 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-09-06T20:36:07,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-09-06T20:36:07,399 INFO [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-09-06T20:36:07,399 DEBUG [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-09-06T20:36:07,400 DEBUG [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-09-06T20:36:07,400 DEBUG [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-09-06T20:36:07,400 DEBUG [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:36:07,408 DEBUG [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:36:07,408 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:36:07,408 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:36:07,409 DEBUG [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-09-06T20:36:07,409 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-09-06T20:36:07,409 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-09-06T20:36:07,410 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:36:07,410 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-09-06T20:36:07,410 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-09-06T20:36:07,410 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-09-06T20:36:07,410 DEBUG [member: 'jenkins-hbase3.apache.org,46803,1694032527898' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-09-06T20:36:07,410 DEBUG [member: 'jenkins-hbase3.apache.org,46803,1694032527898' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-09-06T20:36:07,411 DEBUG [member: 'jenkins-hbase3.apache.org,46803,1694032527898' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-09-06T20:36:07,411 DEBUG [member: 'jenkins-hbase3.apache.org,46803,1694032527898' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-09-06T20:36:07,411 DEBUG [member: 'jenkins-hbase3.apache.org,46803,1694032527898' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,46803,1694032527898' joining acquired barrier for procedure (rolllog) in zk 2023-09-06T20:36:07,412 DEBUG [member: 'jenkins-hbase3.apache.org,46803,1694032527898' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-09-06T20:36:07,412 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:36:07,412 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:36:07,412 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:36:07,413 DEBUG [member: 'jenkins-hbase3.apache.org,46803,1694032527898' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-09-06T20:36:07,413 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:36:07,413 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:36:07,413 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:36:07,414 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:36:07,414 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:36:07,414 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,46803,1694032527898' joining acquired barrier for procedure 'rolllog' on coordinator 2023-09-06T20:36:07,414 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@3aa34697[Count = 0] remaining members to acquire global barrier 2023-09-06T20:36:07,414 DEBUG [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-09-06T20:36:07,414 DEBUG [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-09-06T20:36:07,416 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-09-06T20:36:07,416 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-09-06T20:36:07,416 DEBUG [member: 'jenkins-hbase3.apache.org,46803,1694032527898' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-09-06T20:36:07,416 DEBUG [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-09-06T20:36:07,416 DEBUG [member: 'jenkins-hbase3.apache.org,46803,1694032527898' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-09-06T20:36:07,416 DEBUG [rs(jenkins-hbase3.apache.org,46803,1694032527898)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:36:07,416 INFO [rs(jenkins-hbase3.apache.org,46803,1694032527898)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1694032557728 highest: 1694032557728 on jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:36:07,416 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C46803%2C1694032527898.meta:.meta(num 1694032557711) roll requested 2023-09-06T20:36:07,428 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/WALs/jenkins-hbase3.apache.org,46803,1694032527898/jenkins-hbase3.apache.org%2C46803%2C1694032527898.meta.1694032557712.meta, exclude list is [], retry=0 2023-09-06T20:36:07,431 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36635,DS-c9c70a6a-ab73-4861-a2f6-0e02c09a2252,DISK] 2023-09-06T20:36:07,434 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/WALs/jenkins-hbase3.apache.org,46803,1694032527898/jenkins-hbase3.apache.org%2C46803%2C1694032527898.meta.1694032557711.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/WALs/jenkins-hbase3.apache.org,46803,1694032527898/jenkins-hbase3.apache.org%2C46803%2C1694032527898.meta.1694032557712.meta 2023-09-06T20:36:07,434 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36635,DS-c9c70a6a-ab73-4861-a2f6-0e02c09a2252,DISK]] 2023-09-06T20:36:07,434 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/WALs/jenkins-hbase3.apache.org,46803,1694032527898/jenkins-hbase3.apache.org%2C46803%2C1694032527898.meta.1694032557711.meta is not closed yet, will try archiving it next time 2023-09-06T20:36:07,434 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C46803%2C1694032527898:(num 1694032557728) roll requested 2023-09-06T20:36:07,436 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/WALs/jenkins-hbase3.apache.org,46803,1694032527898/jenkins-hbase3.apache.org%2C46803%2C1694032527898.meta.1694032557711.meta not finished, retry = 0 2023-09-06T20:36:07,445 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/WALs/jenkins-hbase3.apache.org,46803,1694032527898/jenkins-hbase3.apache.org%2C46803%2C1694032527898.1694032557729, exclude list is [], retry=0 2023-09-06T20:36:07,449 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36635,DS-c9c70a6a-ab73-4861-a2f6-0e02c09a2252,DISK] 2023-09-06T20:36:07,450 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/WALs/jenkins-hbase3.apache.org,46803,1694032527898/jenkins-hbase3.apache.org%2C46803%2C1694032527898.1694032557728 with entries=18, filesize=3.44 KB; new WAL /user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/WALs/jenkins-hbase3.apache.org,46803,1694032527898/jenkins-hbase3.apache.org%2C46803%2C1694032527898.1694032557729 2023-09-06T20:36:07,451 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36635,DS-c9c70a6a-ab73-4861-a2f6-0e02c09a2252,DISK]] 2023-09-06T20:36:07,451 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/WALs/jenkins-hbase3.apache.org,46803,1694032527898/jenkins-hbase3.apache.org%2C46803%2C1694032527898.1694032557728 is not closed yet, will try archiving it next time 2023-09-06T20:36:07,451 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/WALs/jenkins-hbase3.apache.org,46803,1694032527898/jenkins-hbase3.apache.org%2C46803%2C1694032527898.1694032547348 to hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/oldWALs/jenkins-hbase3.apache.org%2C46803%2C1694032527898.1694032547348 2023-09-06T20:36:07,456 DEBUG [rs(jenkins-hbase3.apache.org,46803,1694032527898)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-09-06T20:36:07,456 INFO [rs(jenkins-hbase3.apache.org,46803,1694032527898)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1694032557729 on jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:36:07,466 DEBUG [member: 'jenkins-hbase3.apache.org,46803,1694032527898' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-09-06T20:36:07,466 DEBUG [member: 'jenkins-hbase3.apache.org,46803,1694032527898' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,46803,1694032527898' in zk 2023-09-06T20:36:07,468 DEBUG [member: 'jenkins-hbase3.apache.org,46803,1694032527898' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-09-06T20:36:07,468 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:36:07,468 DEBUG [member: 'jenkins-hbase3.apache.org,46803,1694032527898' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:36:07,468 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:36:07,468 DEBUG [member: 'jenkins-hbase3.apache.org,46803,1694032527898' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-09-06T20:36:07,469 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:36:07,470 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:36:07,470 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:36:07,470 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:36:07,471 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:36:07,471 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:36:07,471 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:36:07,472 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:36:07,472 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,46803,1694032527898': 2023-09-06T20:36:07,473 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,46803,1694032527898' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-09-06T20:36:07,473 INFO [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-09-06T20:36:07,473 DEBUG [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-09-06T20:36:07,473 DEBUG [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-09-06T20:36:07,473 DEBUG [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-09-06T20:36:07,473 INFO [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-09-06T20:36:07,476 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-09-06T20:36:07,476 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-09-06T20:36:07,476 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:36:07,476 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:36:07,476 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:36:07,476 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:36:07,476 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:36:07,476 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:36:07,476 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:36:07,477 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:36:07,477 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:36:07,477 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:36:07,478 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:36:07,478 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:36:07,483 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:36:07,483 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:36:07,485 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:36:07,485 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:36:07,485 DEBUG [(jenkins-hbase3.apache.org,42111,1694032524831)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:36:07,486 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-09-06T20:36:07,487 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-09-06T20:36:07,487 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:36:07,487 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:36:07,538 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/WALs/jenkins-hbase3.apache.org,46803,1694032527898/jenkins-hbase3.apache.org%2C46803%2C1694032527898.meta.1694032557711.meta to hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/oldWALs/jenkins-hbase3.apache.org%2C46803%2C1694032527898.meta.1694032557711.meta 2023-09-06T20:36:07,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-09-06T20:36:07,592 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1693859767591_default_test-1694032537394 table=test-1694032537394 type=FLUSH ttl=0 } 2023-09-06T20:36:07,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1693859767592 (current time:1693859767592). 2023-09-06T20:36:07,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:36:07,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1693859767591_default_test-1694032537394 VERSION not specified, setting to 2 2023-09-06T20:36:07,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27d50208, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:36:07,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:36:07,604 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:38218, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:36:07,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:36:07,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:36:07,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] procedure2.ProcedureExecutor(1032): Stored pid=46, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_1693859767591_default_test-1694032537394 table=test-1694032537394 type=FLUSH ttl=0 } 2023-09-06T20:36:07,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1693859767591_default_test-1694032537394 table=test-1694032537394 type=FLUSH ttl=0 }, snapshot procedure id = 46 2023-09-06T20:36:07,610 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_1693859767591_default_test-1694032537394 table=test-1694032537394 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:36:07,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=46 2023-09-06T20:36:07,611 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_1693859767591_default_test-1694032537394 table=test-1694032537394 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:36:07,613 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_1693859767591_default_test-1694032537394 table=test-1694032537394 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:36:07,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=46 2023-09-06T20:36:07,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=46 2023-09-06T20:36:08,021 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_1693859767591_default_test-1694032537394 table=test-1694032537394 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:36:08,021 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE; SnapshotRegionProcedure 7595eee2e2ac3713edf2a8ba161cb7f5}] 2023-09-06T20:36:08,023 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=47, ppid=46, state=RUNNABLE; SnapshotRegionProcedure 7595eee2e2ac3713edf2a8ba161cb7f5 2023-09-06T20:36:08,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46803 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=47 2023-09-06T20:36:08,175 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1694032537394,,1694032537883.7595eee2e2ac3713edf2a8ba161cb7f5. 2023-09-06T20:36:08,175 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.HRegion(2520): Flush status journal for 7595eee2e2ac3713edf2a8ba161cb7f5: 2023-09-06T20:36:08,175 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1694032537394,,1694032537883.7595eee2e2ac3713edf2a8ba161cb7f5. for snapshot_1693859767591_default_test-1694032537394 completed. 2023-09-06T20:36:08,176 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(240): Storing 'test-1694032537394,,1694032537883.7595eee2e2ac3713edf2a8ba161cb7f5.' region-info for snapshot=snapshot_1693859767591_default_test-1694032537394 2023-09-06T20:36:08,176 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:36:08,176 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/default/test-1694032537394/7595eee2e2ac3713edf2a8ba161cb7f5/f/6479ee77687c48c89ac9d7a202cbce01] hfiles 2023-09-06T20:36:08,176 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/default/test-1694032537394/7595eee2e2ac3713edf2a8ba161cb7f5/f/6479ee77687c48c89ac9d7a202cbce01 for snapshot=snapshot_1693859767591_default_test-1694032537394 2023-09-06T20:36:08,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=46 2023-09-06T20:36:08,586 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1694032537394,,1694032537883.7595eee2e2ac3713edf2a8ba161cb7f5. 2023-09-06T20:36:08,586 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=47 2023-09-06T20:36:08,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.HMaster(3995): Remote procedure done, pid=47 2023-09-06T20:36:08,586 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1693859767591_default_test-1694032537394 on region 7595eee2e2ac3713edf2a8ba161cb7f5 2023-09-06T20:36:08,588 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=47, ppid=46, state=RUNNABLE; SnapshotRegionProcedure 7595eee2e2ac3713edf2a8ba161cb7f5 2023-09-06T20:36:08,591 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=47, resume processing ppid=46 2023-09-06T20:36:08,591 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=47, ppid=46, state=SUCCESS; SnapshotRegionProcedure 7595eee2e2ac3713edf2a8ba161cb7f5 in 568 msec 2023-09-06T20:36:08,591 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_1693859767591_default_test-1694032537394 table=test-1694032537394 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:36:08,592 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_1693859767591_default_test-1694032537394 table=test-1694032537394 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:36:08,593 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_1693859767591_default_test-1694032537394 table=test-1694032537394 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:36:08,593 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1693859767591_default_test-1694032537394 2023-09-06T20:36:08,594 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/.hbase-snapshot/.tmp/snapshot_1693859767591_default_test-1694032537394 2023-09-06T20:36:08,605 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_1693859767591_default_test-1694032537394 table=test-1694032537394 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:36:08,611 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_1693859767591_default_test-1694032537394 table=test-1694032537394 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:36:08,612 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/.hbase-snapshot/.tmp/snapshot_1693859767591_default_test-1694032537394 to hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/.hbase-snapshot/snapshot_1693859767591_default_test-1694032537394 2023-09-06T20:36:08,614 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_1693859767591_default_test-1694032537394 table=test-1694032537394 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:36:08,614 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1693859767591_default_test-1694032537394 table=test-1694032537394 type=FLUSH ttl=0 }, snapshot procedure id = 46 2023-09-06T20:36:08,615 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=46, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_1693859767591_default_test-1694032537394 table=test-1694032537394 type=FLUSH ttl=0 } in 1.0070 sec 2023-09-06T20:36:08,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=46 2023-09-06T20:36:08,723 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1693859765232 2023-09-06T20:36:08,723 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-09-06T20:36:08,723 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-09-06T20:36:08,723 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1693859765232_test-1694032537394 2023-09-06T20:36:08,723 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1693859767591_default_test-1694032537394 to hdfs://localhost:42649/backupUT/backup_1693859765232/default/test-1694032537394/ 2023-09-06T20:36:08,723 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-09-06T20:36:08,791 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:42649, inputRoot=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0 2023-09-06T20:36:08,792 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,42111,1694032524831_-1377851620_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:42649/backupUT/backup_1693859765232/default/test-1694032537394, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:42649/backupUT/backup_1693859765232/default/test-1694032537394/.hbase-snapshot/.tmp/snapshot_1693859767591_default_test-1694032537394 2023-09-06T20:36:08,792 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:42649, inputRoot=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0, snapshotDir=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/.hbase-snapshot/snapshot_1693859767591_default_test-1694032537394. 2023-09-06T20:36:08,820 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/.hbase-snapshot/snapshot_1693859767591_default_test-1694032537394 to hdfs://localhost:42649/backupUT/backup_1693859765232/default/test-1694032537394/.hbase-snapshot/.tmp/snapshot_1693859767591_default_test-1694032537394 2023-09-06T20:36:08,890 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:36:08,890 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-09-06T20:36:08,891 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-09-06T20:36:08,892 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-09-06T20:36:09,981 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/hadoop-2816133670912181784.jar 2023-09-06T20:36:09,981 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:36:09,982 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:36:09,982 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:36:09,983 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-09-06T20:36:09,983 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-09-06T20:36:09,984 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-09-06T20:36:09,984 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-09-06T20:36:09,984 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-09-06T20:36:09,985 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-09-06T20:36:09,985 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:36:09,986 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-09-06T20:36:09,987 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-09-06T20:36:09,987 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-09-06T20:36:09,987 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-09-06T20:36:09,988 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-09-06T20:36:09,988 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-09-06T20:36:09,989 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-09-06T20:36:09,989 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-09-06T20:36:09,990 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-09-06T20:36:09,990 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-09-06T20:36:09,991 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-09-06T20:36:09,991 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:36:09,992 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:36:09,992 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:36:09,993 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:36:09,993 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:36:09,994 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:36:09,994 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:36:10,038 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1693859767591_default_test-1694032537394' hfile list 2023-09-06T20:36:10,040 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-09-06T20:36:10,118 WARN [Listener at jenkins-hbase3.apache.org/42325 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/mapreduce.cluster.local.dir/1694032570081/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-09-06T20:36:10,118 WARN [Listener at jenkins-hbase3.apache.org/42325 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/mapreduce.cluster.local.dir/1694032570081/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-09-06T20:36:10,169 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-09-06T20:36:10,196 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/default/test-1694032537394/7595eee2e2ac3713edf2a8ba161cb7f5/f/6479ee77687c48c89ac9d7a202cbce01 output=hdfs://localhost:42649/backupUT/backup_1693859765232/default/test-1694032537394/archive/data/default/test-1694032537394/7595eee2e2ac3713edf2a8ba161cb7f5/f/6479ee77687c48c89ac9d7a202cbce01 2023-09-06T20:36:10,196 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 4.004M/sec 2023-09-06T20:36:11,151 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-09-06T20:36:11,152 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-09-06T20:36:11,159 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1693859767591_default_test-1694032537394 2023-09-06T20:36:11,160 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1693859767591_default_test-1694032537394 finished. 2023-09-06T20:36:11,160 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupSystemTable(1014): test-1694032537394 2023-09-06T20:36:11,167 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1693859765232 2023-09-06T20:36:11,167 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-09-06T20:36:11,573 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:42649/backupUT/backup_1693859765232/.backup.manifest 2023-09-06T20:36:11,573 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.TableBackupClient(358): Backup backup_1693859765232 finished: type=FULL,tablelist=test-1694032537394,targetRootDir=hdfs://localhost:42649/backupUT,startts=1693859767393,completets=1693859771167,bytescopied=0 2023-09-06T20:36:11,573 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-09-06T20:36:11,573 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1693859767591_default_test-1694032537394 2023-09-06T20:36:11,574 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1693859767591_default_test-1694032537394" 2023-09-06T20:36:11,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1693859767591_default_test-1694032537394 2023-09-06T20:36:11,578 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1693859767591_default_test-1694032537394 for backup backup_1693859765232 succeeded. 2023-09-06T20:36:11,579 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-09-06T20:36:11,584 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-09-06T20:36:11,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-09-06T20:36:11,587 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-09-06T20:36:11,589 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-09-06T20:36:11,596 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] impl.TableBackupClient(376): Backup backup_1693859765232 completed. 2023-09-06T20:36:11,596 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:36:11,610 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] backup.TestBackupDelete(125): backup complete 2023-09-06T20:36:11,616 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32a4d805, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:36:11,617 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:36:11,618 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48122, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:36:11,623 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:36:11,624 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:53984, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:36:11,665 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-09-06T20:36:11,675 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:36:11,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032571675 (current time:1694032571675). 2023-09-06T20:36:11,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:36:11,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-09-06T20:36:11,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a07d402, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:36:11,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:36:11,684 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:53996, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:36:11,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:36:11,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:36:11,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] procedure2.ProcedureExecutor(1032): Stored pid=48, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:36:11,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 48 2023-09-06T20:36:11,688 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:36:11,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-09-06T20:36:11,689 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:36:11,691 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:36:11,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-09-06T20:36:11,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-09-06T20:36:12,098 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:36:12,098 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE; SnapshotRegionProcedure c972ac6cf58538c05726e9a461c31a7f}] 2023-09-06T20:36:12,100 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=49, ppid=48, state=RUNNABLE; SnapshotRegionProcedure c972ac6cf58538c05726e9a461c31a7f 2023-09-06T20:36:12,196 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(145): Chore: MemstoreFlusherChore missed its start time 2023-09-06T20:36:12,196 INFO [regionserver/jenkins-hbase3:0.Chore.2 {}] hbase.ScheduledChore(145): Chore: CompactionChecker missed its start time 2023-09-06T20:36:12,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46803 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=49 2023-09-06T20:36:12,252 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f. 2023-09-06T20:36:12,252 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HRegion(2819): Flushing c972ac6cf58538c05726e9a461c31a7f 2/2 column families, dataSize=1.04 KB heapSize=2.20 KB 2023-09-06T20:36:12,257 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/.tmp/meta/b9700242c6224902a3abc02f86459745 is 169, key is trslm:hdfs://localhost:42649/backupUT\x00test-1694032537394/meta:log-roll-map/1693859771163/Put/seqid=0 2023-09-06T20:36:12,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-09-06T20:36:12,662 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=491 B at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/.tmp/meta/b9700242c6224902a3abc02f86459745 2023-09-06T20:36:12,674 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/.tmp/session/311b08101e294fe09cc463b696378b45 is 310, key is session:backup_1693859765232/session:context/1693859771588/Put/seqid=0 2023-09-06T20:36:12,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-09-06T20:36:12,812 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2023-09-06T20:36:12,812 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2023-09-06T20:36:13,079 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=579 B at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/.tmp/session/311b08101e294fe09cc463b696378b45 2023-09-06T20:36:13,087 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/.tmp/meta/b9700242c6224902a3abc02f86459745 as hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/b9700242c6224902a3abc02f86459745 2023-09-06T20:36:13,093 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/b9700242c6224902a3abc02f86459745, entries=5, sequenceid=58, filesize=5.5 K 2023-09-06T20:36:13,095 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/.tmp/session/311b08101e294fe09cc463b696378b45 as hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/311b08101e294fe09cc463b696378b45 2023-09-06T20:36:13,102 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/311b08101e294fe09cc463b696378b45, entries=2, sequenceid=58, filesize=5.3 K 2023-09-06T20:36:13,103 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HRegion(3022): Finished flush of dataSize ~1.04 KB/1070, heapSize ~2.16 KB/2216, currentSize=0 B/0 for c972ac6cf58538c05726e9a461c31a7f in 851ms, sequenceid=58, compaction requested=true 2023-09-06T20:36:13,103 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HRegion(2520): Flush status journal for c972ac6cf58538c05726e9a461c31a7f: 2023-09-06T20:36:13,103 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f. for snapshot_backup_system completed. 2023-09-06T20:36:13,103 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f.' region-info for snapshot=snapshot_backup_system 2023-09-06T20:36:13,103 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:36:13,103 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/87d67d6dd77c4ec7aaac3e7f21bf4489, hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/6f8043cc079d486eb37596a5145dd799, hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/00b44b8959014deaaec4aa483bcdf05c, hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/3dee3ea754fe4dcc839a7959c90f1590, hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/b9700242c6224902a3abc02f86459745] hfiles 2023-09-06T20:36:13,103 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(264): Adding reference for file (1/5): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/87d67d6dd77c4ec7aaac3e7f21bf4489 for snapshot=snapshot_backup_system 2023-09-06T20:36:13,103 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(264): Adding reference for file (2/5): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/6f8043cc079d486eb37596a5145dd799 for snapshot=snapshot_backup_system 2023-09-06T20:36:13,104 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(264): Adding reference for file (3/5): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/00b44b8959014deaaec4aa483bcdf05c for snapshot=snapshot_backup_system 2023-09-06T20:36:13,104 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(264): Adding reference for file (4/5): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/3dee3ea754fe4dcc839a7959c90f1590 for snapshot=snapshot_backup_system 2023-09-06T20:36:13,104 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(264): Adding reference for file (5/5): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/b9700242c6224902a3abc02f86459745 for snapshot=snapshot_backup_system 2023-09-06T20:36:13,104 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/020ff1bc22044c989574f67c5d443447, hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/da3d2282a6d64394ab84e6d230831f40, hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/63d2131a83ba4874a6a538c8a070219b, hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/a78b3e89777d43d393d364a9185f21e9, hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/21f35f677df14d5fae225eba13a89b66, hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/311b08101e294fe09cc463b696378b45] hfiles 2023-09-06T20:36:13,104 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(264): Adding reference for file (1/6): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/020ff1bc22044c989574f67c5d443447 for snapshot=snapshot_backup_system 2023-09-06T20:36:13,104 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(264): Adding reference for file (2/6): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/da3d2282a6d64394ab84e6d230831f40 for snapshot=snapshot_backup_system 2023-09-06T20:36:13,104 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(264): Adding reference for file (3/6): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/63d2131a83ba4874a6a538c8a070219b for snapshot=snapshot_backup_system 2023-09-06T20:36:13,104 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(264): Adding reference for file (4/6): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/a78b3e89777d43d393d364a9185f21e9 for snapshot=snapshot_backup_system 2023-09-06T20:36:13,104 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(264): Adding reference for file (5/6): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/21f35f677df14d5fae225eba13a89b66 for snapshot=snapshot_backup_system 2023-09-06T20:36:13,104 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(264): Adding reference for file (6/6): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/311b08101e294fe09cc463b696378b45 for snapshot=snapshot_backup_system 2023-09-06T20:36:13,510 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f. 2023-09-06T20:36:13,511 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=49 2023-09-06T20:36:13,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.HMaster(3995): Remote procedure done, pid=49 2023-09-06T20:36:13,511 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region c972ac6cf58538c05726e9a461c31a7f 2023-09-06T20:36:13,513 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=49, ppid=48, state=RUNNABLE; SnapshotRegionProcedure c972ac6cf58538c05726e9a461c31a7f 2023-09-06T20:36:13,515 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=49, resume processing ppid=48 2023-09-06T20:36:13,515 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=49, ppid=48, state=SUCCESS; SnapshotRegionProcedure c972ac6cf58538c05726e9a461c31a7f in 1.4160 sec 2023-09-06T20:36:13,515 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:36:13,516 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:36:13,517 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:36:13,517 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-09-06T20:36:13,518 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/.hbase-snapshot/.tmp/snapshot_backup_system 2023-09-06T20:36:13,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-09-06T20:36:13,933 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:36:13,945 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:36:13,946 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/.hbase-snapshot/snapshot_backup_system 2023-09-06T20:36:13,947 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:36:13,947 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 48 2023-09-06T20:36:13,949 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=48, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2620 sec 2023-09-06T20:36:15,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-09-06T20:36:15,801 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-09-06T20:36:15,809 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-09-06T20:36:15,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-09-06T20:36:15,813 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-09-06T20:36:15,813 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-09-06T20:36:15,825 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:36:15,825 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] backup.TestBackupDelete(142): Deleted 0 backups. Total older than 3 days: 0 2023-09-06T20:36:15,852 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6729e939, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:36:15,855 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:36:15,856 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48124, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:36:15,867 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:36:15,868 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:54008, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:36:15,909 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-09-06T20:36:15,917 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:36:15,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032575917 (current time:1694032575917). 2023-09-06T20:36:15,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:36:15,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-09-06T20:36:15,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45e6fa9f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:36:15,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:36:15,934 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:54018, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:36:15,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:36:15,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:36:15,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] procedure2.ProcedureExecutor(1032): Stored pid=50, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:36:15,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 50 2023-09-06T20:36:15,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-09-06T20:36:15,945 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:36:15,946 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:36:15,949 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:36:16,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-09-06T20:36:16,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-09-06T20:36:16,362 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:36:16,362 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE; SnapshotRegionProcedure c972ac6cf58538c05726e9a461c31a7f}] 2023-09-06T20:36:16,364 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=51, ppid=50, state=RUNNABLE; SnapshotRegionProcedure c972ac6cf58538c05726e9a461c31a7f 2023-09-06T20:36:16,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46803 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=51 2023-09-06T20:36:16,516 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f. 2023-09-06T20:36:16,516 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegion(2819): Flushing c972ac6cf58538c05726e9a461c31a7f 2/2 column families, dataSize=186 B heapSize=1000 B 2023-09-06T20:36:16,520 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/.tmp/meta/487b8e7c671142758e505d3821b94dd7 is 64, key is delete_op_row/meta:fam/1694032575914/Put/seqid=0 2023-09-06T20:36:16,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-09-06T20:36:16,925 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=97 B at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/.tmp/meta/487b8e7c671142758e505d3821b94dd7 2023-09-06T20:36:16,931 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 487b8e7c671142758e505d3821b94dd7 2023-09-06T20:36:16,934 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/.tmp/session/98589b7df33b474db1fe919e0ee056a3 is 49, key is activesession:/session:c/1694032575911/Put/seqid=0 2023-09-06T20:36:17,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-09-06T20:36:17,338 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=89 B at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/.tmp/session/98589b7df33b474db1fe919e0ee056a3 2023-09-06T20:36:17,344 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/.tmp/meta/487b8e7c671142758e505d3821b94dd7 as hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/487b8e7c671142758e505d3821b94dd7 2023-09-06T20:36:17,350 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 487b8e7c671142758e505d3821b94dd7 2023-09-06T20:36:17,350 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/487b8e7c671142758e505d3821b94dd7, entries=2, sequenceid=65, filesize=5.1 K 2023-09-06T20:36:17,351 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/.tmp/session/98589b7df33b474db1fe919e0ee056a3 as hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/98589b7df33b474db1fe919e0ee056a3 2023-09-06T20:36:17,357 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/98589b7df33b474db1fe919e0ee056a3, entries=1, sequenceid=65, filesize=4.9 K 2023-09-06T20:36:17,358 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegion(3022): Finished flush of dataSize ~186 B/186, heapSize ~968 B/968, currentSize=0 B/0 for c972ac6cf58538c05726e9a461c31a7f in 841ms, sequenceid=65, compaction requested=true 2023-09-06T20:36:17,358 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegion(2520): Flush status journal for c972ac6cf58538c05726e9a461c31a7f: 2023-09-06T20:36:17,358 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f. for snapshot_backup_system completed. 2023-09-06T20:36:17,358 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f.' region-info for snapshot=snapshot_backup_system 2023-09-06T20:36:17,358 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:36:17,358 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/87d67d6dd77c4ec7aaac3e7f21bf4489, hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/6f8043cc079d486eb37596a5145dd799, hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/00b44b8959014deaaec4aa483bcdf05c, hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/3dee3ea754fe4dcc839a7959c90f1590, hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/b9700242c6224902a3abc02f86459745, hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/487b8e7c671142758e505d3821b94dd7] hfiles 2023-09-06T20:36:17,358 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (1/6): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/87d67d6dd77c4ec7aaac3e7f21bf4489 for snapshot=snapshot_backup_system 2023-09-06T20:36:17,358 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (2/6): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/6f8043cc079d486eb37596a5145dd799 for snapshot=snapshot_backup_system 2023-09-06T20:36:17,358 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (3/6): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/00b44b8959014deaaec4aa483bcdf05c for snapshot=snapshot_backup_system 2023-09-06T20:36:17,358 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (4/6): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/3dee3ea754fe4dcc839a7959c90f1590 for snapshot=snapshot_backup_system 2023-09-06T20:36:17,358 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (5/6): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/b9700242c6224902a3abc02f86459745 for snapshot=snapshot_backup_system 2023-09-06T20:36:17,358 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (6/6): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/487b8e7c671142758e505d3821b94dd7 for snapshot=snapshot_backup_system 2023-09-06T20:36:17,358 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/020ff1bc22044c989574f67c5d443447, hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/da3d2282a6d64394ab84e6d230831f40, hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/63d2131a83ba4874a6a538c8a070219b, hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/a78b3e89777d43d393d364a9185f21e9, hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/21f35f677df14d5fae225eba13a89b66, hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/311b08101e294fe09cc463b696378b45, hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/98589b7df33b474db1fe919e0ee056a3] hfiles 2023-09-06T20:36:17,358 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (1/7): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/020ff1bc22044c989574f67c5d443447 for snapshot=snapshot_backup_system 2023-09-06T20:36:17,358 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (2/7): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/da3d2282a6d64394ab84e6d230831f40 for snapshot=snapshot_backup_system 2023-09-06T20:36:17,358 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (3/7): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/63d2131a83ba4874a6a538c8a070219b for snapshot=snapshot_backup_system 2023-09-06T20:36:17,358 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (4/7): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/a78b3e89777d43d393d364a9185f21e9 for snapshot=snapshot_backup_system 2023-09-06T20:36:17,358 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (5/7): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/21f35f677df14d5fae225eba13a89b66 for snapshot=snapshot_backup_system 2023-09-06T20:36:17,358 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (6/7): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/311b08101e294fe09cc463b696378b45 for snapshot=snapshot_backup_system 2023-09-06T20:36:17,358 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (7/7): hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/98589b7df33b474db1fe919e0ee056a3 for snapshot=snapshot_backup_system 2023-09-06T20:36:17,764 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f. 2023-09-06T20:36:17,764 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=51 2023-09-06T20:36:17,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.HMaster(3995): Remote procedure done, pid=51 2023-09-06T20:36:17,765 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region c972ac6cf58538c05726e9a461c31a7f 2023-09-06T20:36:17,766 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=51, ppid=50, state=RUNNABLE; SnapshotRegionProcedure c972ac6cf58538c05726e9a461c31a7f 2023-09-06T20:36:17,769 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=51, resume processing ppid=50 2023-09-06T20:36:17,769 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:36:17,769 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=51, ppid=50, state=SUCCESS; SnapshotRegionProcedure c972ac6cf58538c05726e9a461c31a7f in 1.4050 sec 2023-09-06T20:36:17,769 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:36:17,770 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:36:17,770 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-09-06T20:36:17,771 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/.hbase-snapshot/.tmp/snapshot_backup_system 2023-09-06T20:36:17,780 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:36:17,793 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:36:17,794 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/.hbase-snapshot/snapshot_backup_system 2023-09-06T20:36:17,795 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:36:17,795 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 50 2023-09-06T20:36:17,797 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=50, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.8590 sec 2023-09-06T20:36:18,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-09-06T20:36:18,058 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupAdminImpl(227): Deleting backup backup_1693859765232 ... 2023-09-06T20:36:18,059 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] util.BackupUtils(435): Delete log files: .backup.manifest 2023-09-06T20:36:18,060 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] util.BackupUtils(435): Delete log files: default 2023-09-06T20:36:18,061 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] util.BackupUtils(443): Trying to cleanup up target dir : backup_1693859765232 2023-09-06T20:36:18,061 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] util.BackupUtils(458): No data has been found in hdfs://localhost:42649/backupUT/backup_1693859765232/default/test-1694032537394. 2023-09-06T20:36:18,062 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] util.BackupUtils(465): hdfs://localhost:42649/backupUT/backup_1693859765232/default is empty, remove it. 2023-09-06T20:36:18,067 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupAdminImpl(269): 0 bulk loaded files out of 0 were deleted 2023-09-06T20:36:18,067 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupSystemTable(433): deleted 0 original bulkload rows 2023-09-06T20:36:18,069 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupAdminImpl(276): Delete backup backup_1693859765232 completed. 2023-09-06T20:36:18,076 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupSystemTable(1014): test-1694032537394 2023-09-06T20:36:18,079 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-09-06T20:36:18,083 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-09-06T20:36:18,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42111 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-09-06T20:36:18,086 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-09-06T20:36:18,086 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-09-06T20:36:18,091 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:36:18,091 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] backup.TestBackupDelete(158): Deleted 1 backups. Total older than 1 days: 1 2023-09-06T20:36:18,108 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] hbase.ResourceChecker(175): after: backup.TestBackupDelete#testBackupPurgeOldBackupsCommand Thread=600 (was 601), OpenFileDescriptor=691 (was 683) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=167 (was 132) - SystemLoadAverage LEAK? -, ProcessCount=170 (was 170), AvailableMemoryMB=2505 (was 2553) 2023-09-06T20:36:18,108 WARN [Listener at jenkins-hbase3.apache.org/42325 {}] hbase.ResourceChecker(130): Thread=600 is superior to 500 2023-09-06T20:36:18,115 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2023-09-06T20:36:18,115 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:36:18,115 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2023-09-06T20:36:18,115 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] util.JVMClusterUtil(257): Found active master hash=1813336176, stopped=false 2023-09-06T20:36:18,116 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] master.ServerManager(910): Cluster shutdown requested of master=jenkins-hbase3.apache.org,42111,1694032524831 2023-09-06T20:36:18,117 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] procedure2.ProcedureExecutor(630): Stopping 2023-09-06T20:36:18,118 DEBUG [Listener at jenkins-hbase3.apache.org/42325 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:36:18,118 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase3.apache.org,46803,1694032527898' ***** 2023-09-06T20:36:18,118 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] regionserver.HRegionServer(2531): STOPPED: Shutdown requested 2023-09-06T20:36:18,118 WARN [RS:0;jenkins-hbase3:46803 {}] util.Sleeper(86): We slept 12946ms instead of 100ms, this is likely due to a long garbage collecting pause and it's usually bad, see http://hbase.apache.org/book.html#trouble.rs.runtime.zkexpired 2023-09-06T20:36:18,118 INFO [RS:0;jenkins-hbase3:46803 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-09-06T20:36:18,131 INFO [RS:0;jenkins-hbase3:46803 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@dfee667{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-09-06T20:36:18,135 INFO [RS:0;jenkins-hbase3:46803 {}] server.AbstractConnector(383): Stopped ServerConnector@3271f28d{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-09-06T20:36:18,135 INFO [RS:0;jenkins-hbase3:46803 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-09-06T20:36:18,136 INFO [RS:0;jenkins-hbase3:46803 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@1a457c7a{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-09-06T20:36:18,137 INFO [RS:0;jenkins-hbase3:46803 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@17ea525b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/hadoop.log.dir/,STOPPED} 2023-09-06T20:36:18,139 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2023-09-06T20:36:18,139 INFO [RS:0;jenkins-hbase3:46803 {}] regionserver.HeapMemoryManager(220): Stopping 2023-09-06T20:36:18,139 INFO [RS:0;jenkins-hbase3:46803 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2023-09-06T20:36:18,139 INFO [RS:0;jenkins-hbase3:46803 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2023-09-06T20:36:18,139 INFO [RS:0;jenkins-hbase3:46803 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2023-09-06T20:36:18,139 INFO [RS:0;jenkins-hbase3:46803 {}] regionserver.HRegionServer(3528): Received CLOSE for 9e465118ef7ff0106801ef31e1e1a961 2023-09-06T20:36:18,140 INFO [RS:0;jenkins-hbase3:46803 {}] regionserver.HRegionServer(3528): Received CLOSE for 7595eee2e2ac3713edf2a8ba161cb7f5 2023-09-06T20:36:18,140 INFO [RS:0;jenkins-hbase3:46803 {}] regionserver.HRegionServer(3528): Received CLOSE for 280ad1b518a781a3a7bb41de290d511c 2023-09-06T20:36:18,140 INFO [RS:0;jenkins-hbase3:46803 {}] regionserver.HRegionServer(3528): Received CLOSE for 67d870a84d8d83d52f8b7cef0eced24d 2023-09-06T20:36:18,140 INFO [RS:0;jenkins-hbase3:46803 {}] regionserver.HRegionServer(3528): Received CLOSE for 15cfd8c09eef178ff8687020ba736ee6 2023-09-06T20:36:18,140 INFO [RS:0;jenkins-hbase3:46803 {}] regionserver.HRegionServer(3528): Received CLOSE for 0489809eda53d4f420cc714a3ab0c1bc 2023-09-06T20:36:18,140 INFO [RS:0;jenkins-hbase3:46803 {}] regionserver.HRegionServer(3528): Received CLOSE for c972ac6cf58538c05726e9a461c31a7f 2023-09-06T20:36:18,140 INFO [RS:0;jenkins-hbase3:46803 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,46803,1694032527898 2023-09-06T20:36:18,140 DEBUG [RS:0;jenkins-hbase3:46803 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:36:18,141 INFO [RS:0;jenkins-hbase3:46803 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2023-09-06T20:36:18,141 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 9e465118ef7ff0106801ef31e1e1a961, disabling compactions & flushes 2023-09-06T20:36:18,141 INFO [RS:0;jenkins-hbase3:46803 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2023-09-06T20:36:18,141 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns2:test-16940325373941,,1694032538629.9e465118ef7ff0106801ef31e1e1a961. 2023-09-06T20:36:18,141 INFO [RS:0;jenkins-hbase3:46803 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2023-09-06T20:36:18,141 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16940325373941,,1694032538629.9e465118ef7ff0106801ef31e1e1a961. 2023-09-06T20:36:18,141 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16940325373941,,1694032538629.9e465118ef7ff0106801ef31e1e1a961. after waiting 0 ms 2023-09-06T20:36:18,141 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16940325373941,,1694032538629.9e465118ef7ff0106801ef31e1e1a961. 2023-09-06T20:36:18,141 INFO [RS:0;jenkins-hbase3:46803 {}] regionserver.HRegionServer(3528): Received CLOSE for 1588230740 2023-09-06T20:36:18,141 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 9e465118ef7ff0106801ef31e1e1a961 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-09-06T20:36:18,141 INFO [RS:0;jenkins-hbase3:46803 {}] regionserver.HRegionServer(1579): Waiting on 8 regions to close 2023-09-06T20:36:18,141 DEBUG [RS:0;jenkins-hbase3:46803 {}] regionserver.HRegionServer(1583): Online Regions={9e465118ef7ff0106801ef31e1e1a961=ns2:test-16940325373941,,1694032538629.9e465118ef7ff0106801ef31e1e1a961., 7595eee2e2ac3713edf2a8ba161cb7f5=test-1694032537394,,1694032537883.7595eee2e2ac3713edf2a8ba161cb7f5., 280ad1b518a781a3a7bb41de290d511c=ns3:test-16940325373942,,1694032539341.280ad1b518a781a3a7bb41de290d511c., 67d870a84d8d83d52f8b7cef0eced24d=ns4:test-16940325373943,,1694032540465.67d870a84d8d83d52f8b7cef0eced24d., 15cfd8c09eef178ff8687020ba736ee6=backup:system_bulk,,1694032543920.15cfd8c09eef178ff8687020ba736ee6., 0489809eda53d4f420cc714a3ab0c1bc=hbase:namespace,,1694032531333.0489809eda53d4f420cc714a3ab0c1bc., 1588230740=hbase:meta,,1.1588230740, c972ac6cf58538c05726e9a461c31a7f=backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f.} 2023-09-06T20:36:18,142 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-09-06T20:36:18,142 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2023-09-06T20:36:18,142 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-09-06T20:36:18,142 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-09-06T20:36:18,142 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-09-06T20:36:18,142 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-09-06T20:36:18,142 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2819): Flushing 1588230740 3/3 column families, dataSize=9.25 KB heapSize=16.20 KB 2023-09-06T20:36:18,143 DEBUG [RS:0;jenkins-hbase3:46803 {}] regionserver.HRegionServer(1609): Waiting on 0489809eda53d4f420cc714a3ab0c1bc, 1588230740, 15cfd8c09eef178ff8687020ba736ee6, 280ad1b518a781a3a7bb41de290d511c, 67d870a84d8d83d52f8b7cef0eced24d, 7595eee2e2ac3713edf2a8ba161cb7f5, 9e465118ef7ff0106801ef31e1e1a961, c972ac6cf58538c05726e9a461c31a7f 2023-09-06T20:36:18,164 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/ns2/test-16940325373941/9e465118ef7ff0106801ef31e1e1a961/.tmp/f/abef88b379ac41eeadf743c1a86ce0bb is 37, key is row10/f:q1/1694032539265/Put/seqid=0 2023-09-06T20:36:18,169 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/ns2/test-16940325373941/9e465118ef7ff0106801ef31e1e1a961/.tmp/f/abef88b379ac41eeadf743c1a86ce0bb 2023-09-06T20:36:18,174 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/hbase/meta/1588230740/.tmp/info/027c936693af4885aabcb2e4db5d1463 is 159, key is ns2:test-16940325373941,,1694032538629.9e465118ef7ff0106801ef31e1e1a961./info:regioninfo/1694032539021/Put/seqid=0 2023-09-06T20:36:18,176 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/ns2/test-16940325373941/9e465118ef7ff0106801ef31e1e1a961/.tmp/f/abef88b379ac41eeadf743c1a86ce0bb as hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/ns2/test-16940325373941/9e465118ef7ff0106801ef31e1e1a961/f/abef88b379ac41eeadf743c1a86ce0bb 2023-09-06T20:36:18,182 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/ns2/test-16940325373941/9e465118ef7ff0106801ef31e1e1a961/f/abef88b379ac41eeadf743c1a86ce0bb, entries=99, sequenceid=103, filesize=8.2 K 2023-09-06T20:36:18,183 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 9e465118ef7ff0106801ef31e1e1a961 in 42ms, sequenceid=103, compaction requested=false 2023-09-06T20:36:18,193 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/ns2/test-16940325373941/9e465118ef7ff0106801ef31e1e1a961/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2023-09-06T20:36:18,193 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:36:18,194 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns2:test-16940325373941,,1694032538629.9e465118ef7ff0106801ef31e1e1a961. 2023-09-06T20:36:18,194 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 9e465118ef7ff0106801ef31e1e1a961: 2023-09-06T20:36:18,195 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:test-16940325373941,,1694032538629.9e465118ef7ff0106801ef31e1e1a961. 2023-09-06T20:36:18,195 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 7595eee2e2ac3713edf2a8ba161cb7f5, disabling compactions & flushes 2023-09-06T20:36:18,195 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region test-1694032537394,,1694032537883.7595eee2e2ac3713edf2a8ba161cb7f5. 2023-09-06T20:36:18,195 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1694032537394,,1694032537883.7595eee2e2ac3713edf2a8ba161cb7f5. 2023-09-06T20:36:18,195 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on test-1694032537394,,1694032537883.7595eee2e2ac3713edf2a8ba161cb7f5. after waiting 0 ms 2023-09-06T20:36:18,195 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region test-1694032537394,,1694032537883.7595eee2e2ac3713edf2a8ba161cb7f5. 2023-09-06T20:36:18,197 INFO [regionserver/jenkins-hbase3:0.Chore.3 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2023-09-06T20:36:18,197 INFO [regionserver/jenkins-hbase3:0.Chore.2 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2023-09-06T20:36:18,200 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/default/test-1694032537394/7595eee2e2ac3713edf2a8ba161cb7f5/recovered.edits/108.seqid, newMaxSeqId=108, maxSeqId=1 2023-09-06T20:36:18,200 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:36:18,201 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed test-1694032537394,,1694032537883.7595eee2e2ac3713edf2a8ba161cb7f5. 2023-09-06T20:36:18,201 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 7595eee2e2ac3713edf2a8ba161cb7f5: 2023-09-06T20:36:18,201 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test-1694032537394,,1694032537883.7595eee2e2ac3713edf2a8ba161cb7f5. 2023-09-06T20:36:18,202 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 280ad1b518a781a3a7bb41de290d511c, disabling compactions & flushes 2023-09-06T20:36:18,202 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns3:test-16940325373942,,1694032539341.280ad1b518a781a3a7bb41de290d511c. 2023-09-06T20:36:18,202 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16940325373942,,1694032539341.280ad1b518a781a3a7bb41de290d511c. 2023-09-06T20:36:18,202 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16940325373942,,1694032539341.280ad1b518a781a3a7bb41de290d511c. after waiting 0 ms 2023-09-06T20:36:18,202 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16940325373942,,1694032539341.280ad1b518a781a3a7bb41de290d511c. 2023-09-06T20:36:18,205 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/ns3/test-16940325373942/280ad1b518a781a3a7bb41de290d511c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-09-06T20:36:18,206 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:36:18,207 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns3:test-16940325373942,,1694032539341.280ad1b518a781a3a7bb41de290d511c. 2023-09-06T20:36:18,207 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 280ad1b518a781a3a7bb41de290d511c: 2023-09-06T20:36:18,207 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns3:test-16940325373942,,1694032539341.280ad1b518a781a3a7bb41de290d511c. 2023-09-06T20:36:18,207 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 67d870a84d8d83d52f8b7cef0eced24d, disabling compactions & flushes 2023-09-06T20:36:18,207 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns4:test-16940325373943,,1694032540465.67d870a84d8d83d52f8b7cef0eced24d. 2023-09-06T20:36:18,207 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16940325373943,,1694032540465.67d870a84d8d83d52f8b7cef0eced24d. 2023-09-06T20:36:18,207 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16940325373943,,1694032540465.67d870a84d8d83d52f8b7cef0eced24d. after waiting 0 ms 2023-09-06T20:36:18,208 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16940325373943,,1694032540465.67d870a84d8d83d52f8b7cef0eced24d. 2023-09-06T20:36:18,211 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/ns4/test-16940325373943/67d870a84d8d83d52f8b7cef0eced24d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-09-06T20:36:18,211 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:36:18,212 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns4:test-16940325373943,,1694032540465.67d870a84d8d83d52f8b7cef0eced24d. 2023-09-06T20:36:18,212 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 67d870a84d8d83d52f8b7cef0eced24d: 2023-09-06T20:36:18,212 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns4:test-16940325373943,,1694032540465.67d870a84d8d83d52f8b7cef0eced24d. 2023-09-06T20:36:18,213 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 15cfd8c09eef178ff8687020ba736ee6, disabling compactions & flushes 2023-09-06T20:36:18,213 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1694032543920.15cfd8c09eef178ff8687020ba736ee6. 2023-09-06T20:36:18,213 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1694032543920.15cfd8c09eef178ff8687020ba736ee6. 2023-09-06T20:36:18,213 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1694032543920.15cfd8c09eef178ff8687020ba736ee6. after waiting 0 ms 2023-09-06T20:36:18,213 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1694032543920.15cfd8c09eef178ff8687020ba736ee6. 2023-09-06T20:36:18,216 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system_bulk/15cfd8c09eef178ff8687020ba736ee6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-09-06T20:36:18,216 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:36:18,217 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system_bulk,,1694032543920.15cfd8c09eef178ff8687020ba736ee6. 2023-09-06T20:36:18,217 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 15cfd8c09eef178ff8687020ba736ee6: 2023-09-06T20:36:18,217 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system_bulk,,1694032543920.15cfd8c09eef178ff8687020ba736ee6. 2023-09-06T20:36:18,218 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 0489809eda53d4f420cc714a3ab0c1bc, disabling compactions & flushes 2023-09-06T20:36:18,218 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:namespace,,1694032531333.0489809eda53d4f420cc714a3ab0c1bc. 2023-09-06T20:36:18,218 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1694032531333.0489809eda53d4f420cc714a3ab0c1bc. 2023-09-06T20:36:18,218 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1694032531333.0489809eda53d4f420cc714a3ab0c1bc. after waiting 0 ms 2023-09-06T20:36:18,218 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1694032531333.0489809eda53d4f420cc714a3ab0c1bc. 2023-09-06T20:36:18,218 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 0489809eda53d4f420cc714a3ab0c1bc 1/1 column families, dataSize=249 B heapSize=1.02 KB 2023-09-06T20:36:18,233 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/hbase/namespace/0489809eda53d4f420cc714a3ab0c1bc/.tmp/info/ad47e4bdfbb6423583170df2e441dc07 is 45, key is default/info:d/1694032532718/Put/seqid=0 2023-09-06T20:36:18,343 DEBUG [RS:0;jenkins-hbase3:46803 {}] regionserver.HRegionServer(1609): Waiting on 0489809eda53d4f420cc714a3ab0c1bc, 1588230740, c972ac6cf58538c05726e9a461c31a7f 2023-09-06T20:36:18,543 DEBUG [RS:0;jenkins-hbase3:46803 {}] regionserver.HRegionServer(1609): Waiting on 0489809eda53d4f420cc714a3ab0c1bc, 1588230740, c972ac6cf58538c05726e9a461c31a7f 2023-09-06T20:36:18,591 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.55 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/hbase/meta/1588230740/.tmp/info/027c936693af4885aabcb2e4db5d1463 2023-09-06T20:36:18,618 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/hbase/meta/1588230740/.tmp/table/03170540a74e48c1b899696f93289ef7 is 59, key is ns2:test-16940325373941/table:state/1694032539030/Put/seqid=0 2023-09-06T20:36:18,638 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/hbase/namespace/0489809eda53d4f420cc714a3ab0c1bc/.tmp/info/ad47e4bdfbb6423583170df2e441dc07 2023-09-06T20:36:18,645 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/hbase/namespace/0489809eda53d4f420cc714a3ab0c1bc/.tmp/info/ad47e4bdfbb6423583170df2e441dc07 as hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/hbase/namespace/0489809eda53d4f420cc714a3ab0c1bc/info/ad47e4bdfbb6423583170df2e441dc07 2023-09-06T20:36:18,651 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/hbase/namespace/0489809eda53d4f420cc714a3ab0c1bc/info/ad47e4bdfbb6423583170df2e441dc07, entries=7, sequenceid=11, filesize=5.1 K 2023-09-06T20:36:18,653 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~249 B/249, heapSize ~1.01 KB/1032, currentSize=0 B/0 for 0489809eda53d4f420cc714a3ab0c1bc in 434ms, sequenceid=11, compaction requested=false 2023-09-06T20:36:18,662 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/hbase/namespace/0489809eda53d4f420cc714a3ab0c1bc/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2023-09-06T20:36:18,663 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:36:18,664 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:namespace,,1694032531333.0489809eda53d4f420cc714a3ab0c1bc. 2023-09-06T20:36:18,664 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 0489809eda53d4f420cc714a3ab0c1bc: 2023-09-06T20:36:18,664 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1694032531333.0489809eda53d4f420cc714a3ab0c1bc. 2023-09-06T20:36:18,665 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing c972ac6cf58538c05726e9a461c31a7f, disabling compactions & flushes 2023-09-06T20:36:18,665 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f. 2023-09-06T20:36:18,665 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f. 2023-09-06T20:36:18,665 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f. after waiting 0 ms 2023-09-06T20:36:18,665 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f. 2023-09-06T20:36:18,665 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing c972ac6cf58538c05726e9a461c31a7f 2/2 column families, dataSize=223 B heapSize=1.01 KB 2023-09-06T20:36:18,671 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/.tmp/meta/acb0ed8055ff41f0a8fc42c656cf57be is 91, key is incrbackupset:hdfs://localhost:42649/backupUT/meta:test-1694032537394/1694032578077/Put/seqid=0 2023-09-06T20:36:18,744 DEBUG [RS:0;jenkins-hbase3:46803 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, c972ac6cf58538c05726e9a461c31a7f 2023-09-06T20:36:18,944 DEBUG [RS:0;jenkins-hbase3:46803 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, c972ac6cf58538c05726e9a461c31a7f 2023-09-06T20:36:19,029 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=714 B at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/hbase/meta/1588230740/.tmp/table/03170540a74e48c1b899696f93289ef7 2023-09-06T20:36:19,037 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/hbase/meta/1588230740/.tmp/info/027c936693af4885aabcb2e4db5d1463 as hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/hbase/meta/1588230740/info/027c936693af4885aabcb2e4db5d1463 2023-09-06T20:36:19,043 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/hbase/meta/1588230740/info/027c936693af4885aabcb2e4db5d1463, entries=70, sequenceid=39, filesize=14.1 K 2023-09-06T20:36:19,044 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/hbase/meta/1588230740/.tmp/table/03170540a74e48c1b899696f93289ef7 as hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/hbase/meta/1588230740/table/03170540a74e48c1b899696f93289ef7 2023-09-06T20:36:19,050 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/hbase/meta/1588230740/table/03170540a74e48c1b899696f93289ef7, entries=14, sequenceid=39, filesize=5.8 K 2023-09-06T20:36:19,051 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3022): Finished flush of dataSize ~9.25 KB/9474, heapSize ~15.92 KB/16304, currentSize=0 B/0 for 1588230740 in 909ms, sequenceid=39, compaction requested=false 2023-09-06T20:36:19,062 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/hbase/meta/1588230740/recovered.edits/42.seqid, newMaxSeqId=42, maxSeqId=1 2023-09-06T20:36:19,062 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:36:19,063 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-09-06T20:36:19,063 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-09-06T20:36:19,063 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-09-06T20:36:19,063 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2023-09-06T20:36:19,076 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=124 B at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/.tmp/meta/acb0ed8055ff41f0a8fc42c656cf57be 2023-09-06T20:36:19,082 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for acb0ed8055ff41f0a8fc42c656cf57be 2023-09-06T20:36:19,085 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/.tmp/session/585a21299cd642ef8d8b5ca624be406e is 59, key is session:backup_1693859765232/session:/1694032578068/DeleteFamily/seqid=0 2023-09-06T20:36:19,144 INFO [RS:0;jenkins-hbase3:46803 {}] regionserver.HRegionServer(1579): Waiting on 1 regions to close 2023-09-06T20:36:19,144 DEBUG [RS:0;jenkins-hbase3:46803 {}] regionserver.HRegionServer(1583): Online Regions={c972ac6cf58538c05726e9a461c31a7f=backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f.} 2023-09-06T20:36:19,144 DEBUG [RS:0;jenkins-hbase3:46803 {}] regionserver.HRegionServer(1609): Waiting on c972ac6cf58538c05726e9a461c31a7f 2023-09-06T20:36:19,344 DEBUG [RS:0;jenkins-hbase3:46803 {}] regionserver.HRegionServer(1609): Waiting on c972ac6cf58538c05726e9a461c31a7f 2023-09-06T20:36:19,490 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=99 B at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/.tmp/session/585a21299cd642ef8d8b5ca624be406e 2023-09-06T20:36:19,496 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 585a21299cd642ef8d8b5ca624be406e 2023-09-06T20:36:19,497 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/.tmp/meta/acb0ed8055ff41f0a8fc42c656cf57be as hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/acb0ed8055ff41f0a8fc42c656cf57be 2023-09-06T20:36:19,503 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for acb0ed8055ff41f0a8fc42c656cf57be 2023-09-06T20:36:19,504 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/meta/acb0ed8055ff41f0a8fc42c656cf57be, entries=2, sequenceid=72, filesize=5.2 K 2023-09-06T20:36:19,505 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/.tmp/session/585a21299cd642ef8d8b5ca624be406e as hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/585a21299cd642ef8d8b5ca624be406e 2023-09-06T20:36:19,512 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 585a21299cd642ef8d8b5ca624be406e 2023-09-06T20:36:19,512 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/session/585a21299cd642ef8d8b5ca624be406e, entries=2, sequenceid=72, filesize=5.2 K 2023-09-06T20:36:19,513 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~223 B/223, heapSize ~1000 B/1000, currentSize=0 B/0 for c972ac6cf58538c05726e9a461c31a7f in 848ms, sequenceid=72, compaction requested=true 2023-09-06T20:36:19,545 DEBUG [RS:0;jenkins-hbase3:46803 {}] regionserver.HRegionServer(1609): Waiting on c972ac6cf58538c05726e9a461c31a7f 2023-09-06T20:36:19,548 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/data/backup/system/c972ac6cf58538c05726e9a461c31a7f/recovered.edits/75.seqid, newMaxSeqId=75, maxSeqId=1 2023-09-06T20:36:19,549 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:36:19,549 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f. 2023-09-06T20:36:19,549 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for c972ac6cf58538c05726e9a461c31a7f: 2023-09-06T20:36:19,550 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system,,1694032542793.c972ac6cf58538c05726e9a461c31a7f. 2023-09-06T20:36:19,745 INFO [RS:0;jenkins-hbase3:46803 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,46803,1694032527898; all regions closed. 2023-09-06T20:36:19,753 DEBUG [RS:0;jenkins-hbase3:46803 {}] wal.AbstractFSWAL(1062): Moved 2 WAL file(s) to /user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/oldWALs 2023-09-06T20:36:19,753 INFO [RS:0;jenkins-hbase3:46803 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C46803%2C1694032527898.meta:.meta(num 1694032557712) 2023-09-06T20:36:19,760 DEBUG [RS:0;jenkins-hbase3:46803 {}] wal.AbstractFSWAL(1062): Moved 3 WAL file(s) to /user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/oldWALs 2023-09-06T20:36:19,760 INFO [RS:0;jenkins-hbase3:46803 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C46803%2C1694032527898:(num 1694032557729) 2023-09-06T20:36:19,760 DEBUG [RS:0;jenkins-hbase3:46803 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:36:19,760 INFO [RS:0;jenkins-hbase3:46803 {}] regionserver.LeaseManager(133): Closed leases 2023-09-06T20:36:19,760 INFO [RS:0;jenkins-hbase3:46803 {}] hbase.ChoreService(370): Chore service for: regionserver/jenkins-hbase3:0 had [ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2023-09-06T20:36:19,761 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-09-06T20:36:19,762 INFO [RS:0;jenkins-hbase3:46803 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.12.81:46803 2023-09-06T20:36:19,776 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [jenkins-hbase3.apache.org,46803,1694032527898] 2023-09-06T20:36:19,776 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing jenkins-hbase3.apache.org,46803,1694032527898; numProcessing=1 2023-09-06T20:36:19,778 INFO [RegionServerTracker-0 {}] master.ServerManager(569): Cluster shutdown set; jenkins-hbase3.apache.org,46803,1694032527898 expired; onlineServers=0 2023-09-06T20:36:19,778 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase3.apache.org,42111,1694032524831' ***** 2023-09-06T20:36:19,778 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2531): STOPPED: Cluster shutdown set; onlineServer=0 2023-09-06T20:36:19,779 WARN [M:0;jenkins-hbase3:42111 {}] util.Sleeper(86): We slept 14552ms instead of 100ms, this is likely due to a long garbage collecting pause and it's usually bad, see http://hbase.apache.org/book.html#trouble.rs.runtime.zkexpired 2023-09-06T20:36:19,779 DEBUG [M:0;jenkins-hbase3:42111 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1024ec41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-09-06T20:36:19,779 INFO [M:0;jenkins-hbase3:42111 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-09-06T20:36:19,823 INFO [M:0;jenkins-hbase3:42111 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@3ad8d793{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-09-06T20:36:19,824 INFO [M:0;jenkins-hbase3:42111 {}] server.AbstractConnector(383): Stopped ServerConnector@164c99cb{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-09-06T20:36:19,824 INFO [M:0;jenkins-hbase3:42111 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-09-06T20:36:19,824 INFO [M:0;jenkins-hbase3:42111 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@30251af4{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-09-06T20:36:19,825 INFO [M:0;jenkins-hbase3:42111 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@1a69b8b4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/hadoop.log.dir/,STOPPED} 2023-09-06T20:36:19,825 INFO [M:0;jenkins-hbase3:42111 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,42111,1694032524831 2023-09-06T20:36:19,826 INFO [M:0;jenkins-hbase3:42111 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,42111,1694032524831; all regions closed. 2023-09-06T20:36:19,826 DEBUG [M:0;jenkins-hbase3:42111 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:36:19,826 INFO [M:0;jenkins-hbase3:42111 {}] master.HMaster(1665): Stopping master jetty server 2023-09-06T20:36:19,826 INFO [M:0;jenkins-hbase3:42111 {}] server.AbstractConnector(383): Stopped ServerConnector@585959c5{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-09-06T20:36:19,827 DEBUG [M:0;jenkins-hbase3:42111 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2023-09-06T20:36:19,827 INFO [M:0;jenkins-hbase3:42111 {}] master.BackupLogCleaner(171): Stopping BackupLogCleaner 2023-09-06T20:36:19,827 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2023-09-06T20:36:19,827 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1694032530055 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1694032530055,5,FailOnTimeoutGroup] 2023-09-06T20:36:19,827 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1694032530049 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1694032530049,5,FailOnTimeoutGroup] 2023-09-06T20:36:19,827 DEBUG [M:0;jenkins-hbase3:42111 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:36:19,827 DEBUG [M:0;jenkins-hbase3:42111 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2023-09-06T20:36:19,827 INFO [M:0;jenkins-hbase3:42111 {}] hbase.ChoreService(370): Chore service for: master/jenkins-hbase3:0 had [] on shutdown 2023-09-06T20:36:19,828 DEBUG [M:0;jenkins-hbase3:42111 {}] master.HMaster(1688): Stopping service threads 2023-09-06T20:36:19,828 INFO [M:0;jenkins-hbase3:42111 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2023-09-06T20:36:19,828 ERROR [M:0;jenkins-hbase3:42111 {}] procedure2.ProcedureExecutor(654): ThreadGroup java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] contains running threads; null: See STDOUT 2023-09-06T20:36:19,830 INFO [M:0;jenkins-hbase3:42111 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2023-09-06T20:36:19,830 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2023-09-06T20:36:19,830 WARN [M:0;jenkins-hbase3:42111 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2023-09-06T20:36:19,830 INFO [M:0;jenkins-hbase3:42111 {}] assignment.AssignmentManager(382): Stopping assignment manager 2023-09-06T20:36:19,830 INFO [M:0;jenkins-hbase3:42111 {}] region.MasterRegion(178): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2023-09-06T20:36:19,831 DEBUG [M:0;jenkins-hbase3:42111 {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-09-06T20:36:19,843 INFO [M:0;jenkins-hbase3:42111 {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:36:19,843 DEBUG [M:0;jenkins-hbase3:42111 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:36:19,843 DEBUG [M:0;jenkins-hbase3:42111 {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-09-06T20:36:19,843 DEBUG [M:0;jenkins-hbase3:42111 {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:36:19,843 INFO [M:0;jenkins-hbase3:42111 {}] regionserver.HRegion(2819): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=195.18 KB heapSize=236.13 KB 2023-09-06T20:36:19,858 DEBUG [M:0;jenkins-hbase3:42111 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/38bf15b3402649d6a08dd9b2a5a74605 is 88, key is hbase:meta,,1/info:sn/1694032530635/Put/seqid=0 2023-09-06T20:36:19,875 INFO [RS:0;jenkins-hbase3:46803 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,46803,1694032527898; zookeeper connection closed. 2023-09-06T20:36:19,876 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6386fdb3 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6386fdb3 2023-09-06T20:36:19,876 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2023-09-06T20:36:20,263 INFO [M:0;jenkins-hbase3:42111 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=530 B at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/38bf15b3402649d6a08dd9b2a5a74605 2023-09-06T20:36:20,291 DEBUG [M:0;jenkins-hbase3:42111 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4065e980b13043359185a3863d0c9a0c is 987, key is \x00\x00\x00\x00\x00\x00\x00\x1D/proc:d/1694032544709/Put/seqid=0 2023-09-06T20:36:20,297 INFO [M:0;jenkins-hbase3:42111 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=194.58 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4065e980b13043359185a3863d0c9a0c 2023-09-06T20:36:20,304 INFO [M:0;jenkins-hbase3:42111 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4065e980b13043359185a3863d0c9a0c 2023-09-06T20:36:20,325 DEBUG [M:0;jenkins-hbase3:42111 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c7070b64f3584134942c6a6d7a129429 is 82, key is jenkins-hbase3.apache.org,46803,1694032527898/rs:state/1694032530069/Put/seqid=0 2023-09-06T20:36:20,735 INFO [M:0;jenkins-hbase3:42111 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c7070b64f3584134942c6a6d7a129429 2023-09-06T20:36:20,742 DEBUG [M:0;jenkins-hbase3:42111 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/38bf15b3402649d6a08dd9b2a5a74605 as hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/38bf15b3402649d6a08dd9b2a5a74605 2023-09-06T20:36:20,748 INFO [M:0;jenkins-hbase3:42111 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/38bf15b3402649d6a08dd9b2a5a74605, entries=8, sequenceid=483, filesize=5.5 K 2023-09-06T20:36:20,749 DEBUG [M:0;jenkins-hbase3:42111 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4065e980b13043359185a3863d0c9a0c as hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4065e980b13043359185a3863d0c9a0c 2023-09-06T20:36:20,755 INFO [M:0;jenkins-hbase3:42111 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4065e980b13043359185a3863d0c9a0c 2023-09-06T20:36:20,755 INFO [M:0;jenkins-hbase3:42111 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4065e980b13043359185a3863d0c9a0c, entries=51, sequenceid=483, filesize=16.1 K 2023-09-06T20:36:20,756 DEBUG [M:0;jenkins-hbase3:42111 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c7070b64f3584134942c6a6d7a129429 as hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c7070b64f3584134942c6a6d7a129429 2023-09-06T20:36:20,763 INFO [M:0;jenkins-hbase3:42111 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42649/user/jenkins/test-data/2fd382d2-9d38-8796-1679-53cab9e1c5e0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c7070b64f3584134942c6a6d7a129429, entries=1, sequenceid=483, filesize=5.1 K 2023-09-06T20:36:20,764 INFO [M:0;jenkins-hbase3:42111 {}] regionserver.HRegion(3022): Finished flush of dataSize ~195.18 KB/199863, heapSize ~235.83 KB/241488, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 921ms, sequenceid=483, compaction requested=false 2023-09-06T20:36:20,770 INFO [M:0;jenkins-hbase3:42111 {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:36:20,770 DEBUG [M:0;jenkins-hbase3:42111 {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-09-06T20:36:20,774 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-09-06T20:36:20,774 INFO [M:0;jenkins-hbase3:42111 {}] flush.MasterFlushTableProcedureManager(85): stop: server shutting down. 2023-09-06T20:36:20,774 INFO [M:0;jenkins-hbase3:42111 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2023-09-06T20:36:20,775 INFO [M:0;jenkins-hbase3:42111 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.12.81:42111 2023-09-06T20:36:20,884 INFO [M:0;jenkins-hbase3:42111 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,42111,1694032524831; zookeeper connection closed. 2023-09-06T20:36:20,886 WARN [Listener at jenkins-hbase3.apache.org/42325 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-09-06T20:36:20,891 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-09-06T20:36:21,026 WARN [BP-259322142-172.31.12.81-1694032521686 heartbeating to localhost/127.0.0.1:42649 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-09-06T20:36:21,026 WARN [BP-259322142-172.31.12.81-1694032521686 heartbeating to localhost/127.0.0.1:42649 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-259322142-172.31.12.81-1694032521686 (Datanode Uuid 687aa97e-2a67-4d56-b087-be677eb3d3a5) service to localhost/127.0.0.1:42649 2023-09-06T20:36:21,028 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/cluster_4bbcbad3-1dd9-986e-e95b-c3cf1f3afe53/dfs/data/data1/current/BP-259322142-172.31.12.81-1694032521686 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-09-06T20:36:21,029 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3b2eaa98-6d47-b9e3-f77d-f7618a69e52e/cluster_4bbcbad3-1dd9-986e-e95b-c3cf1f3afe53/dfs/data/data2/current/BP-259322142-172.31.12.81-1694032521686 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-09-06T20:36:21,062 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-09-06T20:36:21,220 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2023-09-06T20:36:21,220 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] hbase.HBaseTestingUtility(2857): Stopping mini mapreduce cluster... 2023-09-06T20:36:21,235 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-09-06T20:36:21,352 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-09-06T20:36:21,463 ERROR [Thread[Thread-275,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-09-06T20:36:21,464 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-09-06T20:36:21,571 WARN [ApplicationMaster Launcher {}] amlauncher.ApplicationMasterLauncher$LauncherThread(122): org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher$LauncherThread interrupted. Returning. 2023-09-06T20:36:21,579 ERROR [SchedulerEventDispatcher:Event Processor {}] event.EventDispatcher$EventProcessor(61): Returning, interrupted : java.lang.InterruptedException 2023-09-06T20:36:21,580 ERROR [Thread[Thread-285,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-09-06T20:36:21,592 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-09-06T20:36:21,694 ERROR [Thread[Thread-250,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-09-06T20:36:21,695 INFO [Listener at jenkins-hbase3.apache.org/42325 {}] hbase.HBaseTestingUtility(2860): Mini mapreduce cluster stopped