2023-07-22 12:30:48,028 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@175c2241 2023-07-22 12:30:48,046 main DEBUG Took 0.015143 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2023-07-22 12:30:48,047 main DEBUG PluginManager 'Core' found 129 plugins 2023-07-22 12:30:48,047 main DEBUG PluginManager 'Level' found 0 plugins 2023-07-22 12:30:48,048 main DEBUG PluginManager 'Lookup' found 16 plugins 2023-07-22 12:30:48,050 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:30:48,060 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2023-07-22 12:30:48,076 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:30:48,078 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:30:48,078 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:30:48,079 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:30:48,079 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:30:48,079 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:30:48,080 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:30:48,080 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:30:48,081 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:30:48,081 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:30:48,082 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:30:48,082 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:30:48,082 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:30:48,083 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:30:48,083 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:30:48,083 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:30:48,084 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:30:48,084 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:30:48,085 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:30:48,085 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:30:48,085 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:30:48,086 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:30:48,086 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:30:48,086 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:30:48,087 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:30:48,087 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2023-07-22 12:30:48,088 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:30:48,091 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2023-07-22 12:30:48,093 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2023-07-22 12:30:48,093 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2023-07-22 12:30:48,095 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2023-07-22 12:30:48,095 main DEBUG PluginManager 'Converter' found 47 plugins 2023-07-22 12:30:48,110 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2023-07-22 12:30:48,114 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2023-07-22 12:30:48,119 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2023-07-22 12:30:48,119 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2023-07-22 12:30:48,120 main DEBUG createAppenders(={Console}) 2023-07-22 12:30:48,121 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@175c2241 initialized 2023-07-22 12:30:48,121 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@175c2241 2023-07-22 12:30:48,121 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@175c2241 OK. 2023-07-22 12:30:48,122 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2023-07-22 12:30:48,122 main DEBUG OutputStream closed 2023-07-22 12:30:48,122 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2023-07-22 12:30:48,122 main DEBUG Appender DefaultConsole-1 stopped with status true 2023-07-22 12:30:48,123 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@2145b572 OK 2023-07-22 12:30:48,174 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586 2023-07-22 12:30:48,176 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=StatusLogger 2023-07-22 12:30:48,177 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=ContextSelector 2023-07-22 12:30:48,178 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name= 2023-07-22 12:30:48,179 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.directory 2023-07-22 12:30:48,179 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2023-07-22 12:30:48,179 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.zookeeper 2023-07-22 12:30:48,180 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2023-07-22 12:30:48,180 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2023-07-22 12:30:48,180 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2023-07-22 12:30:48,181 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase 2023-07-22 12:30:48,181 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop 2023-07-22 12:30:48,181 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2023-07-22 12:30:48,181 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2023-07-22 12:30:48,182 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2023-07-22 12:30:48,182 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2023-07-22 12:30:48,182 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2023-07-22 12:30:48,183 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Appenders,name=Console 2023-07-22 12:30:48,186 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-07-22 12:30:48,187 main DEBUG Reconfiguration complete for context[name=1b6d3586] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.6.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@49912c99) with optional ClassLoader: null 2023-07-22 12:30:48,187 main DEBUG Shutdown hook enabled. Registering a new one. 2023-07-22 12:30:48,188 main DEBUG LoggerContext[name=1b6d3586, org.apache.logging.log4j.core.LoggerContext@49912c99] started OK. 2023-07-22T12:30:48,223 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.backup.TestFullRestore timeout: 13 mins 2023-07-22 12:30:48,226 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2023-07-22 12:30:48,227 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-07-22T12:30:48,582 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7 2023-07-22T12:30:48,587 DEBUG [Time-limited test {}] impl.BackupManager(127): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner. Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager.Added master procedure manager: org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-07-22T12:30:48,589 DEBUG [Time-limited test {}] impl.BackupManager(157): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager. Added region observer: org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:30:48,590 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2023-07-22T12:30:48,593 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/cluster_3df5f3c2-318c-5fd6-9f69-2531a0a95eef, deleteOnExit=true 2023-07-22T12:30:48,594 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2023-07-22T12:30:48,594 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/test.cache.data in system properties and HBase conf 2023-07-22T12:30:48,595 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/hadoop.tmp.dir in system properties and HBase conf 2023-07-22T12:30:48,595 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/hadoop.log.dir in system properties and HBase conf 2023-07-22T12:30:48,595 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir in system properties and HBase conf 2023-07-22T12:30:48,596 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-07-22T12:30:48,596 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-07-22T12:30:48,700 WARN [Time-limited test {}] util.NativeCodeLoader(62): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2023-07-22T12:30:49,149 DEBUG [Time-limited test {}] fs.HFileSystem(308): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-07-22T12:30:49,154 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-07-22T12:30:49,154 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-07-22T12:30:49,155 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-07-22T12:30:49,155 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-07-22T12:30:49,156 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-07-22T12:30:49,156 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-07-22T12:30:49,157 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-07-22T12:30:49,157 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/dfs.journalnode.edits.dir in system properties and HBase conf 2023-07-22T12:30:49,158 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-07-22T12:30:49,158 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/nfs.dump.dir in system properties and HBase conf 2023-07-22T12:30:49,159 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/java.io.tmpdir in system properties and HBase conf 2023-07-22T12:30:49,159 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/dfs.journalnode.edits.dir in system properties and HBase conf 2023-07-22T12:30:49,160 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-07-22T12:30:49,160 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2023-07-22T12:30:49,631 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-07-22T12:30:49,635 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-07-22T12:30:49,945 WARN [Time-limited test {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2023-07-22T12:30:50,137 INFO [Time-limited test {}] log.Slf4jLog(67): Logging to org.apache.logging.slf4j.Log4jLogger@71ec88cc via org.mortbay.log.Slf4jLog 2023-07-22T12:30:50,157 WARN [Time-limited test {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-07-22T12:30:50,206 INFO [Time-limited test {}] log.Slf4jLog(67): jetty-6.1.26 2023-07-22T12:30:50,242 INFO [Time-limited test {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/java.io.tmpdir/Jetty_localhost_46627_hdfs____igpckj/webapp 2023-07-22T12:30:50,406 INFO [Time-limited test {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:46627 2023-07-22T12:30:50,427 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-07-22T12:30:50,427 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-07-22T12:30:50,934 WARN [Listener at localhost/34745 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-07-22T12:30:51,010 WARN [Listener at localhost/34745 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-07-22T12:30:51,033 WARN [Listener at localhost/34745 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-07-22T12:30:51,041 INFO [Listener at localhost/34745 {}] log.Slf4jLog(67): jetty-6.1.26 2023-07-22T12:30:51,047 INFO [Listener at localhost/34745 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/java.io.tmpdir/Jetty_localhost_32871_datanode____.w41jye/webapp 2023-07-22T12:30:51,156 INFO [Listener at localhost/34745 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:32871 2023-07-22T12:30:51,472 WARN [Listener at localhost/39427 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-07-22T12:30:51,750 WARN [Thread-52 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-07-22T12:30:51,966 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x46c11c84fe9b2456: Processing first storage report for DS-711135b1-7150-45cc-b6bd-e4af6531d48e from datanode 781397b1-b4c2-433c-adfa-c2f66f3a63ff 2023-07-22T12:30:51,968 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x46c11c84fe9b2456: from storage DS-711135b1-7150-45cc-b6bd-e4af6531d48e node DatanodeRegistration(127.0.0.1:39117, datanodeUuid=781397b1-b4c2-433c-adfa-c2f66f3a63ff, infoPort=33503, infoSecurePort=0, ipcPort=39427, storageInfo=lv=-57;cid=testClusterID;nsid=1186014937;c=1690029049712), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2023-07-22T12:30:51,968 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x46c11c84fe9b2456: Processing first storage report for DS-fd20bc4c-3b40-4064-941f-ab224adb55ea from datanode 781397b1-b4c2-433c-adfa-c2f66f3a63ff 2023-07-22T12:30:51,968 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x46c11c84fe9b2456: from storage DS-fd20bc4c-3b40-4064-941f-ab224adb55ea node DatanodeRegistration(127.0.0.1:39117, datanodeUuid=781397b1-b4c2-433c-adfa-c2f66f3a63ff, infoPort=33503, infoSecurePort=0, ipcPort=39427, storageInfo=lv=-57;cid=testClusterID;nsid=1186014937;c=1690029049712), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-07-22T12:30:52,053 DEBUG [Listener at localhost/39427 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7 2023-07-22T12:30:52,141 INFO [Listener at localhost/39427 {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/cluster_3df5f3c2-318c-5fd6-9f69-2531a0a95eef/zookeeper_0, clientPort=62657, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/cluster_3df5f3c2-318c-5fd6-9f69-2531a0a95eef/zookeeper_0/version-2, dataDirSize=424 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/cluster_3df5f3c2-318c-5fd6-9f69-2531a0a95eef/zookeeper_0/version-2, dataLogSize=424 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, serverId=0 2023-07-22T12:30:52,164 INFO [Listener at localhost/39427 {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=62657 2023-07-22T12:30:52,175 INFO [Listener at localhost/39427 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:30:52,177 INFO [Listener at localhost/39427 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:30:52,824 INFO [Listener at localhost/39427 {}] util.FSUtils(461): Created version file at hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947 with version=8 2023-07-22T12:30:52,825 INFO [Listener at localhost/39427 {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging 2023-07-22T12:30:52,848 DEBUG [Listener at localhost/39427 {}] hbase.LocalHBaseCluster(134): Setting Master Port to random. 2023-07-22T12:30:52,848 DEBUG [Listener at localhost/39427 {}] hbase.LocalHBaseCluster(141): Setting RegionServer Port to random. 2023-07-22T12:30:52,848 DEBUG [Listener at localhost/39427 {}] hbase.LocalHBaseCluster(151): Setting RS InfoServer Port to random. 2023-07-22T12:30:52,848 DEBUG [Listener at localhost/39427 {}] hbase.LocalHBaseCluster(159): Setting Master InfoServer Port to random. 2023-07-22T12:30:53,066 DEBUG [Listener at localhost/39427 {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2023-07-22T12:30:53,230 INFO [Listener at localhost/39427 {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2023-07-22T12:30:53,920 INFO [Listener at localhost/39427 {}] client.ConnectionUtils(127): master/jenkins-hbase3:0 server-side Connection retries=45 2023-07-22T12:30:53,968 INFO [Listener at localhost/39427 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-07-22T12:30:53,969 INFO [Listener at localhost/39427 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-07-22T12:30:53,969 INFO [Listener at localhost/39427 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-07-22T12:30:53,969 INFO [Listener at localhost/39427 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-07-22T12:30:53,969 INFO [Listener at localhost/39427 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-07-22T12:30:54,144 INFO [Listener at localhost/39427 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-07-22T12:30:54,255 DEBUG [Listener at localhost/39427 {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2023-07-22T12:30:54,261 INFO [Listener at localhost/39427 {}] ipc.NettyRpcServer(177): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-07-22T12:30:54,318 DEBUG [Listener at localhost/39427 {}] channel.DefaultChannelId(79): -Dio.netty.processId: 7649 (auto-detected) 2023-07-22T12:30:54,320 DEBUG [Listener at localhost/39427 {}] channel.DefaultChannelId(101): -Dio.netty.machineId: 02:42:1c:ff:fe:e2:13:ea (auto-detected) 2023-07-22T12:30:54,359 INFO [Listener at localhost/39427 {}] ipc.NettyRpcServer(144): Bind to /172.31.12.81:37923 2023-07-22T12:30:54,374 INFO [Listener at localhost/39427 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:30:54,376 INFO [Listener at localhost/39427 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:30:54,394 INFO [Listener at localhost/39427 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=master:37923 connecting to ZooKeeper ensemble=127.0.0.1:62657 2023-07-22T12:30:54,447 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:379230x0, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-07-22T12:30:54,451 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): master:37923-0x100005e31340000 connected 2023-07-22T12:30:54,494 DEBUG [Listener at localhost/39427 {}] zookeeper.ZKUtil(113): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-07-22T12:30:54,498 DEBUG [Listener at localhost/39427 {}] zookeeper.ZKUtil(113): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-07-22T12:30:54,503 DEBUG [Listener at localhost/39427 {}] zookeeper.ZKUtil(113): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-07-22T12:30:54,515 DEBUG [Listener at localhost/39427 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37923 2023-07-22T12:30:54,516 DEBUG [Listener at localhost/39427 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37923 2023-07-22T12:30:54,516 DEBUG [Listener at localhost/39427 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37923 2023-07-22T12:30:54,518 DEBUG [Listener at localhost/39427 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37923 2023-07-22T12:30:54,518 DEBUG [Listener at localhost/39427 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37923 2023-07-22T12:30:54,576 INFO [Listener at localhost/39427 {}] log.Log(170): Logging initialized @7301ms to org.apache.hbase.thirdparty.org.eclipse.jetty.util.log.Slf4jLog 2023-07-22T12:30:54,726 INFO [Listener at localhost/39427 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-07-22T12:30:54,727 INFO [Listener at localhost/39427 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-07-22T12:30:54,728 INFO [Listener at localhost/39427 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-07-22T12:30:54,730 INFO [Listener at localhost/39427 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2023-07-22T12:30:54,730 INFO [Listener at localhost/39427 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-07-22T12:30:54,730 INFO [Listener at localhost/39427 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-07-22T12:30:54,734 INFO [Listener at localhost/39427 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-07-22T12:30:54,876 INFO [Listener at localhost/39427 {}] http.HttpServer(1219): Jetty bound to port 44729 2023-07-22T12:30:54,878 INFO [Listener at localhost/39427 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-07-22T12:30:54,912 INFO [Listener at localhost/39427 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-07-22T12:30:54,917 INFO [Listener at localhost/39427 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@6ffeb5f7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/hadoop.log.dir/,AVAILABLE} 2023-07-22T12:30:54,918 INFO [Listener at localhost/39427 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-07-22T12:30:54,919 INFO [Listener at localhost/39427 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@36677652{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-07-22T12:30:54,990 INFO [Listener at localhost/39427 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-07-22T12:30:55,003 INFO [Listener at localhost/39427 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-07-22T12:30:55,003 INFO [Listener at localhost/39427 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-07-22T12:30:55,005 INFO [Listener at localhost/39427 {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2023-07-22T12:30:55,013 INFO [Listener at localhost/39427 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-07-22T12:30:56,249 INFO [Listener at localhost/39427 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@2baba16b{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-07-22T12:30:56,263 INFO [Listener at localhost/39427 {}] server.AbstractConnector(333): Started ServerConnector@4e8aff63{HTTP/1.1, (http/1.1)}{0.0.0.0:44729} 2023-07-22T12:30:56,264 INFO [Listener at localhost/39427 {}] server.Server(415): Started @8988ms 2023-07-22T12:30:56,267 INFO [Listener at localhost/39427 {}] master.HMaster(484): hbase.rootdir=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947, hbase.cluster.distributed=false 2023-07-22T12:30:56,328 INFO [Listener at localhost/39427 {}] client.ConnectionUtils(127): regionserver/jenkins-hbase3:0 server-side Connection retries=45 2023-07-22T12:30:56,329 INFO [Listener at localhost/39427 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-07-22T12:30:56,329 INFO [Listener at localhost/39427 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-07-22T12:30:56,329 INFO [Listener at localhost/39427 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-07-22T12:30:56,329 INFO [Listener at localhost/39427 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-07-22T12:30:56,330 INFO [Listener at localhost/39427 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-07-22T12:30:56,340 INFO [Listener at localhost/39427 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-07-22T12:30:56,343 INFO [Listener at localhost/39427 {}] ipc.NettyRpcServer(177): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-07-22T12:30:56,347 INFO [Listener at localhost/39427 {}] ipc.NettyRpcServer(144): Bind to /172.31.12.81:43639 2023-07-22T12:30:56,350 INFO [Listener at localhost/39427 {}] hfile.BlockCacheFactory(127): Allocating BlockCache size=782.40 MB, blockSize=64 KB 2023-07-22T12:30:56,365 DEBUG [Listener at localhost/39427 {}] mob.MobFileCache(120): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2023-07-22T12:30:56,366 INFO [Listener at localhost/39427 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:30:56,370 INFO [Listener at localhost/39427 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:30:56,372 INFO [Listener at localhost/39427 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=regionserver:43639 connecting to ZooKeeper ensemble=127.0.0.1:62657 2023-07-22T12:30:56,378 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): regionserver:436390x0, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-07-22T12:30:56,379 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): regionserver:43639-0x100005e31340001 connected 2023-07-22T12:30:56,380 DEBUG [Listener at localhost/39427 {}] zookeeper.ZKUtil(113): regionserver:43639-0x100005e31340001, quorum=127.0.0.1:62657, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-07-22T12:30:56,381 DEBUG [Listener at localhost/39427 {}] zookeeper.ZKUtil(113): regionserver:43639-0x100005e31340001, quorum=127.0.0.1:62657, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-07-22T12:30:56,382 DEBUG [Listener at localhost/39427 {}] zookeeper.ZKUtil(113): regionserver:43639-0x100005e31340001, quorum=127.0.0.1:62657, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-07-22T12:30:56,386 DEBUG [Listener at localhost/39427 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43639 2023-07-22T12:30:56,390 DEBUG [Listener at localhost/39427 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43639 2023-07-22T12:30:56,394 DEBUG [Listener at localhost/39427 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43639 2023-07-22T12:30:56,395 DEBUG [Listener at localhost/39427 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43639 2023-07-22T12:30:56,395 DEBUG [Listener at localhost/39427 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43639 2023-07-22T12:30:56,401 INFO [Listener at localhost/39427 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-07-22T12:30:56,401 INFO [Listener at localhost/39427 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-07-22T12:30:56,401 INFO [Listener at localhost/39427 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-07-22T12:30:56,403 INFO [Listener at localhost/39427 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2023-07-22T12:30:56,403 INFO [Listener at localhost/39427 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-07-22T12:30:56,403 INFO [Listener at localhost/39427 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-07-22T12:30:56,403 INFO [Listener at localhost/39427 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-07-22T12:30:56,406 INFO [Listener at localhost/39427 {}] http.HttpServer(1219): Jetty bound to port 33077 2023-07-22T12:30:56,406 INFO [Listener at localhost/39427 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-07-22T12:30:56,415 INFO [Listener at localhost/39427 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-07-22T12:30:56,416 INFO [Listener at localhost/39427 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@727639c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/hadoop.log.dir/,AVAILABLE} 2023-07-22T12:30:56,416 INFO [Listener at localhost/39427 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-07-22T12:30:56,416 INFO [Listener at localhost/39427 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@3cbef607{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-07-22T12:30:56,429 INFO [Listener at localhost/39427 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-07-22T12:30:56,431 INFO [Listener at localhost/39427 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-07-22T12:30:56,431 INFO [Listener at localhost/39427 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-07-22T12:30:56,431 INFO [Listener at localhost/39427 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2023-07-22T12:30:56,435 INFO [Listener at localhost/39427 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-07-22T12:30:56,441 INFO [Listener at localhost/39427 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@59d42308{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-07-22T12:30:56,442 INFO [Listener at localhost/39427 {}] server.AbstractConnector(333): Started ServerConnector@82c78f1{HTTP/1.1, (http/1.1)}{0.0.0.0:33077} 2023-07-22T12:30:56,442 INFO [Listener at localhost/39427 {}] server.Server(415): Started @9167ms 2023-07-22T12:30:56,450 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-07-22T12:30:56,469 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@6b055907{HTTP/1.1, (http/1.1)}{0.0.0.0:39151} 2023-07-22T12:30:56,469 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(415): Started @9194ms 2023-07-22T12:30:56,469 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2387): Adding backup master ZNode /1/backup-masters/jenkins-hbase3.apache.org,37923,1690029053050 2023-07-22T12:30:56,482 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): regionserver:43639-0x100005e31340001, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-07-22T12:30:56,483 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-07-22T12:30:56,484 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Set watcher on existing znode=/1/backup-masters/jenkins-hbase3.apache.org,37923,1690029053050 2023-07-22T12:30:56,485 DEBUG [M:0;jenkins-hbase3:37923 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;jenkins-hbase3:37923 2023-07-22T12:30:56,507 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): regionserver:43639-0x100005e31340001, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-07-22T12:30:56,507 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-07-22T12:30:56,507 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): regionserver:43639-0x100005e31340001, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:30:56,507 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:30:56,508 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Set watcher on existing znode=/1/master 2023-07-22T12:30:56,511 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /1/backup-masters/jenkins-hbase3.apache.org,37923,1690029053050 from backup master directory 2023-07-22T12:30:56,513 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Set watcher on existing znode=/1/master 2023-07-22T12:30:56,517 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): regionserver:43639-0x100005e31340001, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-07-22T12:30:56,518 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/backup-masters/jenkins-hbase3.apache.org,37923,1690029053050 2023-07-22T12:30:56,518 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-07-22T12:30:56,521 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-07-22T12:30:56,521 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=jenkins-hbase3.apache.org,37923,1690029053050 2023-07-22T12:30:56,525 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 352, initial count 0 2023-07-22T12:30:56,530 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 391, initial count 0 2023-07-22T12:30:56,627 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] util.FSUtils(610): Created cluster ID file at hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase.id with ID: edd49369-874b-436d-a3ea-023ee76cc4c0 2023-07-22T12:30:56,683 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:30:56,704 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): regionserver:43639-0x100005e31340001, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:30:56,705 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:30:56,762 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(355): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:30:56,764 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2023-07-22T12:30:56,790 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(270): ClientProtocol::create wrong number of arguments, should be hadoop 3.2 or below 2023-07-22T12:30:56,790 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(276): ClientProtocol::create wrong number of arguments, should be hadoop 2.x 2023-07-22T12:30:56,792 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(285): can not find SHOULD_REPLICATE flag, should be hadoop 2.x java.lang.IllegalArgumentException: No enum constant org.apache.hadoop.fs.CreateFlag.SHOULD_REPLICATE at java.lang.Enum.valueOf(Enum.java:238) ~[?:1.8.0_362] at org.apache.hadoop.fs.CreateFlag.valueOf(CreateFlag.java:63) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.loadShouldReplicateFlag(FanOutOneBlockAsyncDFSOutputHelper.java:283) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.(FanOutOneBlockAsyncDFSOutputHelper.java:310) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:140) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:950) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:573) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:570) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-07-22T12:30:56,803 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:141) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:950) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:573) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:570) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-07-22T12:30:56,804 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-07-22T12:30:57,278 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7093): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/MasterData/data/master/store 2023-07-22T12:30:57,703 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2023-07-22T12:30:57,703 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:30:57,704 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-07-22T12:30:57,704 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:30:57,704 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:30:57,704 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-07-22T12:30:57,704 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:30:57,704 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:30:57,704 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-07-22T12:30:57,706 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(232): failed to clean up initializing flag: hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/MasterData/data/master/store/.initializing 2023-07-22T12:30:57,707 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(202): WALDir=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/MasterData/WALs/jenkins-hbase3.apache.org,37923,1690029053050 2023-07-22T12:30:57,717 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-07-22T12:30:57,731 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C37923%2C1690029053050, suffix=, logDir=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/MasterData/WALs/jenkins-hbase3.apache.org,37923,1690029053050, archiveDir=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/MasterData/oldWALs, maxLogs=10 2023-07-22T12:30:57,760 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/MasterData/WALs/jenkins-hbase3.apache.org,37923,1690029053050/jenkins-hbase3.apache.org%2C37923%2C1690029053050.1690029057738, exclude list is [], retry=0 2023-07-22T12:30:57,786 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK] 2023-07-22T12:30:57,793 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(123): Hadoop 3.2 and below use unshaded protobuf. java.lang.ClassNotFoundException: org.apache.hadoop.thirdparty.protobuf.MessageLite at java.net.URLClassLoader.findClass(URLClassLoader.java:387) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:418) ~[?:1.8.0_362] at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ~[?:1.8.0_362] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.ProtobufDecoder.(ProtobufDecoder.java:118) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.processWriteBlockResponse(FanOutOneBlockAsyncDFSOutputHelper.java:346) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$100(FanOutOneBlockAsyncDFSOutputHelper.java:120) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$4.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:430) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:185) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:35) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hadoop.hbase.util.NettyFutureUtils.addListener(NettyFutureUtils.java:52) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:424) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$300(FanOutOneBlockAsyncDFSOutputHelper.java:120) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:482) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:477) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:653) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:691) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:489) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:397) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.4.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-07-22T12:30:57,864 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/MasterData/WALs/jenkins-hbase3.apache.org,37923,1690029053050/jenkins-hbase3.apache.org%2C37923%2C1690029053050.1690029057738 2023-07-22T12:30:57,865 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK]] 2023-07-22T12:30:57,866 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7254): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:30:57,866 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:30:57,872 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7296): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:30:57,873 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7299): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:30:57,939 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:30:57,976 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2023-07-22T12:30:57,982 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:30:57,985 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-07-22T12:30:57,985 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:30:57,988 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2023-07-22T12:30:57,988 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:30:57,989 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:30:57,989 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:30:57,992 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2023-07-22T12:30:57,992 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:30:57,993 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:30:57,993 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:30:57,996 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2023-07-22T12:30:57,996 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:30:57,997 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:30:58,001 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:30:58,002 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:30:58,016 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2023-07-22T12:30:58,020 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1076): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:30:58,023 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:30:58,024 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1093): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10378963360, jitterRate=-0.03338371217250824}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2023-07-22T12:30:58,031 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(992): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2023-07-22T12:30:58,032 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2023-07-22T12:30:58,065 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53e03a64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:30:58,120 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(859): No meta location available on zookeeper, skip migrating... 2023-07-22T12:30:58,142 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2023-07-22T12:30:58,142 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(564): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2023-07-22T12:30:58,145 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2023-07-22T12:30:58,146 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(584): Recovered RegionProcedureStore lease in 1 msec 2023-07-22T12:30:58,153 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(598): Loaded RegionProcedureStore in 6 msec 2023-07-22T12:30:58,153 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2023-07-22T12:30:58,187 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2023-07-22T12:30:58,203 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Unable to get data of znode /1/balancer because node does not exist (not necessarily an error) 2023-07-22T12:30:58,206 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/balancer already deleted, retry=false 2023-07-22T12:30:58,209 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(150): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2023-07-22T12:30:58,210 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Unable to get data of znode /1/normalizer because node does not exist (not necessarily an error) 2023-07-22T12:30:58,211 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/normalizer already deleted, retry=false 2023-07-22T12:30:58,213 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2023-07-22T12:30:58,220 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Unable to get data of znode /1/switch/split because node does not exist (not necessarily an error) 2023-07-22T12:30:58,222 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/split already deleted, retry=false 2023-07-22T12:30:58,223 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Unable to get data of znode /1/switch/merge because node does not exist (not necessarily an error) 2023-07-22T12:30:58,224 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/merge already deleted, retry=false 2023-07-22T12:30:58,240 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Unable to get data of znode /1/snapshot-cleanup because node does not exist (not necessarily an error) 2023-07-22T12:30:58,241 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/snapshot-cleanup already deleted, retry=false 2023-07-22T12:30:58,246 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-07-22T12:30:58,246 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): regionserver:43639-0x100005e31340001, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-07-22T12:30:58,246 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:30:58,246 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): regionserver:43639-0x100005e31340001, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:30:58,247 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(803): Active/primary master=jenkins-hbase3.apache.org,37923,1690029053050, sessionid=0x100005e31340000, setting cluster-up flag (Was=false) 2023-07-22T12:30:58,258 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2023-07-22T12:30:58,267 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:30:58,267 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): regionserver:43639-0x100005e31340001, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:30:58,272 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/flush-table-proc/acquired, /1/flush-table-proc/reached, /1/flush-table-proc/abort 2023-07-22T12:30:58,273 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,37923,1690029053050 2023-07-22T12:30:58,276 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:30:58,276 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): regionserver:43639-0x100005e31340001, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:30:58,282 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/rolllog-proc/acquired, /1/rolllog-proc/reached, /1/rolllog-proc/abort 2023-07-22T12:30:58,284 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,37923,1690029053050 2023-07-22T12:30:58,288 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:30:58,288 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): regionserver:43639-0x100005e31340001, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:30:58,294 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/online-snapshot/acquired, /1/online-snapshot/reached, /1/online-snapshot/abort 2023-07-22T12:30:58,295 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,37923,1690029053050 2023-07-22T12:30:58,357 DEBUG [RS:0;jenkins-hbase3:43639 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;jenkins-hbase3:43639 2023-07-22T12:30:58,359 INFO [RS:0;jenkins-hbase3:43639 {}] regionserver.HRegionServer(997): ClusterId : edd49369-874b-436d-a3ea-023ee76cc4c0 2023-07-22T12:30:58,361 INFO [RS:0;jenkins-hbase3:43639 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2023-07-22T12:30:58,363 DEBUG [RS:0;jenkins-hbase3:43639 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2023-07-22T12:30:58,369 DEBUG [RS:0;jenkins-hbase3:43639 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2023-07-22T12:30:58,369 DEBUG [RS:0;jenkins-hbase3:43639 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2023-07-22T12:30:58,373 DEBUG [RS:0;jenkins-hbase3:43639 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2023-07-22T12:30:58,373 DEBUG [RS:0;jenkins-hbase3:43639 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2023-07-22T12:30:58,376 DEBUG [RS:0;jenkins-hbase3:43639 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2023-07-22T12:30:58,377 DEBUG [RS:0;jenkins-hbase3:43639 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fa558e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:30:58,379 DEBUG [RS:0;jenkins-hbase3:43639 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68f49944, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-07-22T12:30:58,384 INFO [RS:0;jenkins-hbase3:43639 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2023-07-22T12:30:58,384 INFO [RS:0;jenkins-hbase3:43639 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2023-07-22T12:30:58,384 DEBUG [RS:0;jenkins-hbase3:43639 {}] regionserver.HRegionServer(1079): About to register with Master. 2023-07-22T12:30:58,387 INFO [RS:0;jenkins-hbase3:43639 {}] regionserver.HRegionServer(3023): reportForDuty to master=jenkins-hbase3.apache.org,37923,1690029053050 with isa=jenkins-hbase3.apache.org/172.31.12.81:43639, startcode=1690029056327 2023-07-22T12:30:58,405 DEBUG [RS:0;jenkins-hbase3:43639 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-07-22T12:30:58,439 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1033): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2023-07-22T12:30:58,449 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(574): slop=0.2 2023-07-22T12:30:58,456 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(285): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2023-07-22T12:30:58,467 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: jenkins-hbase3.apache.org,37923,1690029053050 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2023-07-22T12:30:58,473 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-07-22T12:30:58,473 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-07-22T12:30:58,473 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-07-22T12:30:58,473 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-07-22T12:30:58,474 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/jenkins-hbase3:0, corePoolSize=10, maxPoolSize=10 2023-07-22T12:30:58,474 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:30:58,474 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-07-22T12:30:58,474 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:30:58,485 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:34387, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-07-22T12:30:58,491 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1690029088491 2023-07-22T12:30:58,494 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2023-07-22T12:30:58,499 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37923 {}] ipc.MetricsHBaseServer(144): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3180) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:588) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17393) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-07-22T12:30:58,500 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2023-07-22T12:30:58,503 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2023-07-22T12:30:58,504 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2023-07-22T12:30:58,510 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:30:58,511 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-07-22T12:30:58,517 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2023-07-22T12:30:58,517 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2023-07-22T12:30:58,518 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2023-07-22T12:30:58,519 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2023-07-22T12:30:58,519 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2023-07-22T12:30:58,520 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-07-22T12:30:58,524 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2023-07-22T12:30:58,528 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2023-07-22T12:30:58,529 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2023-07-22T12:30:58,529 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2023-07-22T12:30:58,529 INFO [PEWorker-1 {}] regionserver.HRegion(7075): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947 2023-07-22T12:30:58,539 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ReadOnlyZKClient(139): Connect 0x28a3126a to 127.0.0.1:62657 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:30:58,551 DEBUG [RS:0;jenkins-hbase3:43639 {}] regionserver.HRegionServer(3042): Master is not running yet 2023-07-22T12:30:58,551 WARN [RS:0;jenkins-hbase3:43639 {}] regionserver.HRegionServer(1088): reportForDuty failed; sleeping 100 ms and then retrying. 2023-07-22T12:30:58,556 DEBUG [PEWorker-1 {}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:30:58,559 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@535f047f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:30:58,559 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-07-22T12:30:58,562 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-07-22T12:30:58,563 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2023-07-22T12:30:58,564 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2023-07-22T12:30:58,565 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-07-22T12:30:58,566 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:30:58,566 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1690029058566,5,FailOnTimeoutGroup] 2023-07-22T12:30:58,567 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-07-22T12:30:58,567 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1690029058567,5,FailOnTimeoutGroup] 2023-07-22T12:30:58,567 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-07-22T12:30:58,567 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-07-22T12:30:58,567 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1634): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2023-07-22T12:30:58,569 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2023-07-22T12:30:58,569 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-07-22T12:30:58,569 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2023-07-22T12:30:58,569 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:30:58,570 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-07-22T12:30:58,571 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-07-22T12:30:58,573 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-07-22T12:30:58,573 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:30:58,574 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-07-22T12:30:58,576 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/hbase/meta/1588230740 2023-07-22T12:30:58,576 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/hbase/meta/1588230740 2023-07-22T12:30:58,579 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-07-22T12:30:58,582 DEBUG [PEWorker-1 {}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-07-22T12:30:58,586 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:30:58,586 INFO [PEWorker-1 {}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10044335840, jitterRate=-0.06454832851886749}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-07-22T12:30:58,589 DEBUG [PEWorker-1 {}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-07-22T12:30:58,589 DEBUG [PEWorker-1 {}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-07-22T12:30:58,589 INFO [PEWorker-1 {}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-07-22T12:30:58,589 DEBUG [PEWorker-1 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-07-22T12:30:58,589 DEBUG [PEWorker-1 {}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-07-22T12:30:58,589 DEBUG [PEWorker-1 {}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-07-22T12:30:58,591 INFO [PEWorker-1 {}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-07-22T12:30:58,591 DEBUG [PEWorker-1 {}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-07-22T12:30:58,594 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2023-07-22T12:30:58,594 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2023-07-22T12:30:58,603 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2023-07-22T12:30:58,617 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2023-07-22T12:30:58,624 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2023-07-22T12:30:58,653 INFO [RS:0;jenkins-hbase3:43639 {}] regionserver.HRegionServer(3023): reportForDuty to master=jenkins-hbase3.apache.org,37923,1690029053050 with isa=jenkins-hbase3.apache.org/172.31.12.81:43639, startcode=1690029056327 2023-07-22T12:30:58,658 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37923 {}] master.ServerManager(403): Registering regionserver=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:30:58,670 DEBUG [RS:0;jenkins-hbase3:43639 {}] regionserver.HRegionServer(1705): Config from master: hbase.rootdir=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947 2023-07-22T12:30:58,670 DEBUG [RS:0;jenkins-hbase3:43639 {}] regionserver.HRegionServer(1705): Config from master: fs.defaultFS=hdfs://localhost:34745 2023-07-22T12:30:58,670 DEBUG [RS:0;jenkins-hbase3:43639 {}] regionserver.HRegionServer(1705): Config from master: hbase.master.info.port=44729 2023-07-22T12:30:58,676 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2023-07-22T12:30:58,677 DEBUG [RS:0;jenkins-hbase3:43639 {}] zookeeper.ZKUtil(111): regionserver:43639-0x100005e31340001, quorum=127.0.0.1:62657, baseZNode=/1 Set watcher on existing znode=/1/rs/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:30:58,677 WARN [RS:0;jenkins-hbase3:43639 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-07-22T12:30:58,677 INFO [RS:0;jenkins-hbase3:43639 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-07-22T12:30:58,677 DEBUG [RS:0;jenkins-hbase3:43639 {}] regionserver.HRegionServer(2060): logDir=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:30:58,680 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [jenkins-hbase3.apache.org,43639,1690029056327] 2023-07-22T12:30:58,696 DEBUG [RS:0;jenkins-hbase3:43639 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2023-07-22T12:30:58,706 INFO [RS:0;jenkins-hbase3:43639 {}] regionserver.MetricsRegionServerWrapperImpl(165): Computing regionserver metrics every 5000 milliseconds 2023-07-22T12:30:58,720 INFO [RS:0;jenkins-hbase3:43639 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=782.4 M, globalMemStoreLimitLowMark=743.3 M, Offheap=false 2023-07-22T12:30:58,725 INFO [RS:0;jenkins-hbase3:43639 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2023-07-22T12:30:58,725 INFO [RS:0;jenkins-hbase3:43639 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2023-07-22T12:30:58,726 INFO [RS:0;jenkins-hbase3:43639 {}] regionserver.HRegionServer$CompactionChecker(1947): CompactionChecker runs every PT1S 2023-07-22T12:30:58,736 INFO [RS:0;jenkins-hbase3:43639 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2023-07-22T12:30:58,736 DEBUG [RS:0;jenkins-hbase3:43639 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:30:58,736 DEBUG [RS:0;jenkins-hbase3:43639 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:30:58,736 DEBUG [RS:0;jenkins-hbase3:43639 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:30:58,736 DEBUG [RS:0;jenkins-hbase3:43639 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:30:58,736 DEBUG [RS:0;jenkins-hbase3:43639 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:30:58,737 DEBUG [RS:0;jenkins-hbase3:43639 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-07-22T12:30:58,737 DEBUG [RS:0;jenkins-hbase3:43639 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:30:58,737 DEBUG [RS:0;jenkins-hbase3:43639 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:30:58,737 DEBUG [RS:0;jenkins-hbase3:43639 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:30:58,737 DEBUG [RS:0;jenkins-hbase3:43639 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:30:58,737 DEBUG [RS:0;jenkins-hbase3:43639 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:30:58,737 DEBUG [RS:0;jenkins-hbase3:43639 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-07-22T12:30:58,738 INFO [RS:0;jenkins-hbase3:43639 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2023-07-22T12:30:58,738 INFO [RS:0;jenkins-hbase3:43639 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2023-07-22T12:30:58,738 INFO [RS:0;jenkins-hbase3:43639 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2023-07-22T12:30:58,738 INFO [RS:0;jenkins-hbase3:43639 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2023-07-22T12:30:58,739 INFO [RS:0;jenkins-hbase3:43639 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,43639,1690029056327-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-07-22T12:30:58,754 INFO [RS:0;jenkins-hbase3:43639 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2023-07-22T12:30:58,757 INFO [RS:0;jenkins-hbase3:43639 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,43639,1690029056327-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2023-07-22T12:30:58,773 INFO [RS:0;jenkins-hbase3:43639 {}] regionserver.Replication(204): jenkins-hbase3.apache.org,43639,1690029056327 started 2023-07-22T12:30:58,774 INFO [RS:0;jenkins-hbase3:43639 {}] regionserver.HRegionServer(1747): Serving as jenkins-hbase3.apache.org,43639,1690029056327, RpcServer on jenkins-hbase3.apache.org/172.31.12.81:43639, sessionid=0x100005e31340001 2023-07-22T12:30:58,774 DEBUG [RS:0;jenkins-hbase3:43639 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2023-07-22T12:30:58,774 DEBUG [RS:0;jenkins-hbase3:43639 {}] flush.RegionServerFlushTableProcedureManager(106): Start region server flush procedure manager jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:30:58,774 DEBUG [RS:0;jenkins-hbase3:43639 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,43639,1690029056327' 2023-07-22T12:30:58,774 DEBUG [RS:0;jenkins-hbase3:43639 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/flush-table-proc/abort' 2023-07-22T12:30:58,775 WARN [jenkins-hbase3:37923 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-07-22T12:30:58,775 DEBUG [RS:0;jenkins-hbase3:43639 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/flush-table-proc/acquired' 2023-07-22T12:30:58,776 DEBUG [RS:0;jenkins-hbase3:43639 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2023-07-22T12:30:58,776 DEBUG [RS:0;jenkins-hbase3:43639 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2023-07-22T12:30:58,776 DEBUG [RS:0;jenkins-hbase3:43639 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,43639,1690029056327' 2023-07-22T12:30:58,776 DEBUG [RS:0;jenkins-hbase3:43639 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:30:58,776 DEBUG [RS:0;jenkins-hbase3:43639 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:30:58,777 INFO [RS:0;jenkins-hbase3:43639 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2023-07-22T12:30:58,777 DEBUG [RS:0;jenkins-hbase3:43639 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2023-07-22T12:30:58,777 DEBUG [RS:0;jenkins-hbase3:43639 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2023-07-22T12:30:58,777 DEBUG [RS:0;jenkins-hbase3:43639 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:30:58,777 DEBUG [RS:0;jenkins-hbase3:43639 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,43639,1690029056327' 2023-07-22T12:30:58,777 DEBUG [RS:0;jenkins-hbase3:43639 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/online-snapshot/abort' 2023-07-22T12:30:58,777 DEBUG [RS:0;jenkins-hbase3:43639 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/online-snapshot/acquired' 2023-07-22T12:30:58,778 DEBUG [RS:0;jenkins-hbase3:43639 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2023-07-22T12:30:58,778 INFO [RS:0;jenkins-hbase3:43639 {}] quotas.RegionServerRpcQuotaManager(63): Quota support disabled 2023-07-22T12:30:58,778 INFO [RS:0;jenkins-hbase3:43639 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2023-07-22T12:30:58,888 INFO [RS:0;jenkins-hbase3:43639 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-07-22T12:30:58,891 INFO [RS:0;jenkins-hbase3:43639 {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C43639%2C1690029056327, suffix=, logDir=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327, archiveDir=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/oldWALs, maxLogs=32 2023-07-22T12:30:58,903 DEBUG [RS:0;jenkins-hbase3:43639 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029058892, exclude list is [], retry=0 2023-07-22T12:30:58,908 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK] 2023-07-22T12:30:58,912 INFO [RS:0;jenkins-hbase3:43639 {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029058892 2023-07-22T12:30:58,912 DEBUG [RS:0;jenkins-hbase3:43639 {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK]] 2023-07-22T12:30:59,026 DEBUG [jenkins-hbase3:37923 {}] assignment.AssignmentManager(2303): Processing assignQueue; systemServersCount=1, allServersCount=1 2023-07-22T12:30:59,034 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:30:59,039 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,43639,1690029056327, state=OPENING 2023-07-22T12:30:59,072 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2023-07-22T12:30:59,076 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:30:59,076 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): regionserver:43639-0x100005e31340001, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:30:59,077 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-07-22T12:30:59,078 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-07-22T12:30:59,080 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,43639,1690029056327}] 2023-07-22T12:30:59,271 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(721): New admin connection to jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:30:59,274 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=AdminService, sasl=false 2023-07-22T12:30:59,277 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:47188, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2023-07-22T12:30:59,289 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(130): Open hbase:meta,,1.1588230740 2023-07-22T12:30:59,290 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-07-22T12:30:59,291 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2023-07-22T12:30:59,294 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta, suffix=.meta, logDir=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327, archiveDir=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/oldWALs, maxLogs=32 2023-07-22T12:30:59,308 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029059296.meta, exclude list is [], retry=0 2023-07-22T12:30:59,311 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK] 2023-07-22T12:30:59,316 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029059296.meta 2023-07-22T12:30:59,316 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK]] 2023-07-22T12:30:59,316 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7254): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:30:59,321 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:30:59,322 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2023-07-22T12:30:59,339 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7955): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2023-07-22T12:30:59,339 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(393): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2023-07-22T12:30:59,344 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2023-07-22T12:30:59,344 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:30:59,345 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7296): checking encryption for 1588230740 2023-07-22T12:30:59,345 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7299): checking classloading for 1588230740 2023-07-22T12:30:59,348 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-07-22T12:30:59,350 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-07-22T12:30:59,350 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:30:59,351 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-07-22T12:30:59,351 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-07-22T12:30:59,352 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-07-22T12:30:59,352 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:30:59,353 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-07-22T12:30:59,353 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-07-22T12:30:59,355 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-07-22T12:30:59,355 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:30:59,356 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-07-22T12:30:59,358 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/hbase/meta/1588230740 2023-07-22T12:30:59,360 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/hbase/meta/1588230740 2023-07-22T12:30:59,363 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-07-22T12:30:59,366 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-07-22T12:30:59,367 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10512479840, jitterRate=-0.020949020981788635}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-07-22T12:30:59,368 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-07-22T12:30:59,382 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2552): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1690029059263 2023-07-22T12:30:59,403 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2579): Finished post open deploy task for hbase:meta,,1.1588230740 2023-07-22T12:30:59,404 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(158): Opened hbase:meta,,1.1588230740 2023-07-22T12:30:59,405 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:30:59,407 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,43639,1690029056327, state=OPEN 2023-07-22T12:30:59,410 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-07-22T12:30:59,410 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): regionserver:43639-0x100005e31340001, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-07-22T12:30:59,410 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-07-22T12:30:59,410 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-07-22T12:30:59,414 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=3, resume processing ppid=2 2023-07-22T12:30:59,414 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,43639,1690029056327 in 330 msec 2023-07-22T12:30:59,425 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=2, resume processing ppid=1 2023-07-22T12:30:59,425 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 812 msec 2023-07-22T12:30:59,431 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 1.0720 sec 2023-07-22T12:30:59,431 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1062): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1690029059431, completionTime=-1 2023-07-22T12:30:59,431 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ServerManager(830): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2023-07-22T12:30:59,431 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1629): Joining cluster... 2023-07-22T12:30:59,636 DEBUG [hconnection-0x4c100189-shared-pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:30:59,640 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:47196, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:30:59,655 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1641): Number of RegionServers=1 2023-07-22T12:30:59,655 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1690029119655 2023-07-22T12:30:59,655 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1690029179655 2023-07-22T12:30:59,655 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1648): Joined the cluster in 223 msec 2023-07-22T12:30:59,677 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,37923,1690029053050-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2023-07-22T12:30:59,677 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,37923,1690029053050-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2023-07-22T12:30:59,677 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,37923,1690029053050-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2023-07-22T12:30:59,679 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-jenkins-hbase3:37923, period=300000, unit=MILLISECONDS is enabled. 2023-07-22T12:30:59,680 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2023-07-22T12:30:59,685 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] janitor.CatalogJanitor(175): 2023-07-22T12:30:59,699 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2023-07-22T12:30:59,701 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2367): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-07-22T12:30:59,711 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1033): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2023-07-22T12:30:59,715 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:30:59,716 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:30:59,718 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:31:00,136 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 595143905c5328f814fba73ad7017c76, NAME => 'hbase:namespace,,1690029059700.595143905c5328f814fba73ad7017c76.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947 2023-07-22T12:31:00,547 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:namespace,,1690029059700.595143905c5328f814fba73ad7017c76.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:31:00,547 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1668): Closing 595143905c5328f814fba73ad7017c76, disabling compactions & flushes 2023-07-22T12:31:00,547 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:namespace,,1690029059700.595143905c5328f814fba73ad7017c76. 2023-07-22T12:31:00,547 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1690029059700.595143905c5328f814fba73ad7017c76. 2023-07-22T12:31:00,547 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1690029059700.595143905c5328f814fba73ad7017c76. after waiting 0 ms 2023-07-22T12:31:00,547 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1690029059700.595143905c5328f814fba73ad7017c76. 2023-07-22T12:31:00,547 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1909): Closed hbase:namespace,,1690029059700.595143905c5328f814fba73ad7017c76. 2023-07-22T12:31:00,547 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1622): Region close journal for 595143905c5328f814fba73ad7017c76: 2023-07-22T12:31:00,550 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:31:00,563 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1690029059700.595143905c5328f814fba73ad7017c76.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1690029060551"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690029060551"}]},"ts":"1690029060551"} 2023-07-22T12:31:00,596 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:31:00,598 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:31:00,603 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029060598"}]},"ts":"1690029060598"} 2023-07-22T12:31:00,607 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2023-07-22T12:31:00,613 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=595143905c5328f814fba73ad7017c76, ASSIGN}] 2023-07-22T12:31:00,615 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=595143905c5328f814fba73ad7017c76, ASSIGN 2023-07-22T12:31:00,617 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=595143905c5328f814fba73ad7017c76, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,43639,1690029056327; forceNewPlan=false, retain=false 2023-07-22T12:31:00,767 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=595143905c5328f814fba73ad7017c76, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:00,771 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 595143905c5328f814fba73ad7017c76, server=jenkins-hbase3.apache.org,43639,1690029056327}] 2023-07-22T12:31:00,931 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(130): Open hbase:namespace,,1690029059700.595143905c5328f814fba73ad7017c76. 2023-07-22T12:31:00,931 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7254): Opening region: {ENCODED => 595143905c5328f814fba73ad7017c76, NAME => 'hbase:namespace,,1690029059700.595143905c5328f814fba73ad7017c76.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:31:00,932 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:31:00,933 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 595143905c5328f814fba73ad7017c76 2023-07-22T12:31:00,933 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(888): Instantiated hbase:namespace,,1690029059700.595143905c5328f814fba73ad7017c76.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:31:00,933 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7296): checking encryption for 595143905c5328f814fba73ad7017c76 2023-07-22T12:31:00,933 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7299): checking classloading for 595143905c5328f814fba73ad7017c76 2023-07-22T12:31:00,935 INFO [StoreOpener-595143905c5328f814fba73ad7017c76-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 595143905c5328f814fba73ad7017c76 2023-07-22T12:31:00,937 INFO [StoreOpener-595143905c5328f814fba73ad7017c76-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 595143905c5328f814fba73ad7017c76 columnFamilyName info 2023-07-22T12:31:00,937 DEBUG [StoreOpener-595143905c5328f814fba73ad7017c76-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:31:00,938 INFO [StoreOpener-595143905c5328f814fba73ad7017c76-1 {}] regionserver.HStore(324): Store=595143905c5328f814fba73ad7017c76/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:31:00,940 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/hbase/namespace/595143905c5328f814fba73ad7017c76 2023-07-22T12:31:00,940 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/hbase/namespace/595143905c5328f814fba73ad7017c76 2023-07-22T12:31:00,944 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1076): writing seq id for 595143905c5328f814fba73ad7017c76 2023-07-22T12:31:00,947 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/hbase/namespace/595143905c5328f814fba73ad7017c76/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:31:00,948 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1093): Opened 595143905c5328f814fba73ad7017c76; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11792637760, jitterRate=0.09827497601509094}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-07-22T12:31:00,949 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(992): Region open journal for 595143905c5328f814fba73ad7017c76: 2023-07-22T12:31:00,951 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2552): Post open deploy tasks for hbase:namespace,,1690029059700.595143905c5328f814fba73ad7017c76., pid=6, masterSystemTime=1690029060925 2023-07-22T12:31:00,954 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2579): Finished post open deploy task for hbase:namespace,,1690029059700.595143905c5328f814fba73ad7017c76. 2023-07-22T12:31:00,954 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(158): Opened hbase:namespace,,1690029059700.595143905c5328f814fba73ad7017c76. 2023-07-22T12:31:00,955 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=595143905c5328f814fba73ad7017c76, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:00,962 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=6, resume processing ppid=5 2023-07-22T12:31:00,962 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 595143905c5328f814fba73ad7017c76, server=jenkins-hbase3.apache.org,43639,1690029056327 in 188 msec 2023-07-22T12:31:00,965 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=5, resume processing ppid=4 2023-07-22T12:31:00,965 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=595143905c5328f814fba73ad7017c76, ASSIGN in 349 msec 2023-07-22T12:31:00,966 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:31:00,967 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029060966"}]},"ts":"1690029060966"} 2023-07-22T12:31:00,969 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2023-07-22T12:31:00,972 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:31:00,975 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 1.2720 sec 2023-07-22T12:31:01,016 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Set watcher on znode that does not yet exist, /1/namespace 2023-07-22T12:31:01,017 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/namespace 2023-07-22T12:31:01,017 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): regionserver:43639-0x100005e31340001, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:31:01,017 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:31:01,076 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1033): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2023-07-22T12:31:01,094 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-07-22T12:31:01,099 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 35 msec 2023-07-22T12:31:01,111 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1033): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2023-07-22T12:31:01,125 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-07-22T12:31:01,130 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 18 msec 2023-07-22T12:31:01,138 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/default 2023-07-22T12:31:01,140 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/hbase 2023-07-22T12:31:01,140 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1192): Master has completed initialization 4.619sec 2023-07-22T12:31:01,142 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2023-07-22T12:31:01,144 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2023-07-22T12:31:01,145 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2023-07-22T12:31:01,146 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2023-07-22T12:31:01,146 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2023-07-22T12:31:01,148 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,37923,1690029053050-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-07-22T12:31:01,149 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,37923,1690029053050-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2023-07-22T12:31:01,162 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1295): Balancer post startup initialization complete, took 0 seconds 2023-07-22T12:31:01,163 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2023-07-22T12:31:01,190 DEBUG [Listener at localhost/39427 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x4069941d to 127.0.0.1:62657 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:31:01,195 DEBUG [Listener at localhost/39427 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@704a280e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:31:01,200 DEBUG [Listener at localhost/39427 {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2023-07-22T12:31:01,200 DEBUG [Listener at localhost/39427 {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2023-07-22T12:31:01,214 DEBUG [hconnection-0x69245e7b-shared-pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:31:01,225 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:36668, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:31:01,235 INFO [Listener at localhost/39427 {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=jenkins-hbase3.apache.org,37923,1690029053050 2023-07-22T12:31:01,235 INFO [Listener at localhost/39427 {}] hbase.HBaseTestingUtility(2786): Starting mini mapreduce cluster... 2023-07-22T12:31:01,235 INFO [Listener at localhost/39427 {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/test.cache.data in system properties and HBase conf 2023-07-22T12:31:01,235 INFO [Listener at localhost/39427 {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/hadoop.tmp.dir in system properties and HBase conf 2023-07-22T12:31:01,235 INFO [Listener at localhost/39427 {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/hadoop.log.dir in system properties and HBase conf 2023-07-22T12:31:01,236 INFO [Listener at localhost/39427 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir in system properties and HBase conf 2023-07-22T12:31:01,236 INFO [Listener at localhost/39427 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-07-22T12:31:01,236 INFO [Listener at localhost/39427 {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-07-22T12:31:01,236 INFO [Listener at localhost/39427 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-07-22T12:31:01,236 INFO [Listener at localhost/39427 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-07-22T12:31:01,236 INFO [Listener at localhost/39427 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-07-22T12:31:01,236 INFO [Listener at localhost/39427 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-07-22T12:31:01,236 INFO [Listener at localhost/39427 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-07-22T12:31:01,236 INFO [Listener at localhost/39427 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-07-22T12:31:01,236 INFO [Listener at localhost/39427 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-07-22T12:31:01,236 INFO [Listener at localhost/39427 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/dfs.journalnode.edits.dir in system properties and HBase conf 2023-07-22T12:31:01,236 INFO [Listener at localhost/39427 {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-07-22T12:31:01,236 INFO [Listener at localhost/39427 {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/nfs.dump.dir in system properties and HBase conf 2023-07-22T12:31:01,236 INFO [Listener at localhost/39427 {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/java.io.tmpdir in system properties and HBase conf 2023-07-22T12:31:01,237 INFO [Listener at localhost/39427 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/dfs.journalnode.edits.dir in system properties and HBase conf 2023-07-22T12:31:01,237 INFO [Listener at localhost/39427 {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-07-22T12:31:01,237 INFO [Listener at localhost/39427 {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2023-07-22T12:31:02,647 WARN [Thread-249 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-07-22T12:31:03,003 INFO [Thread-249 {}] log.Slf4jLog(67): jetty-6.1.26 2023-07-22T12:31:03,009 INFO [Thread-249 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/jobhistory to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_33785_jobhistory____.j6uw0y/webapp 2023-07-22T12:31:03,165 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices as a root resource class 2023-07-22T12:31:03,166 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver as a provider class 2023-07-22T12:31:03,166 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-07-22T12:31:03,169 INFO [Thread-249 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-07-22T12:31:03,225 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2023-07-22T12:31:03,226 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2023-07-22T12:31:03,235 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2023-07-22T12:31:03,235 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2023-07-22T12:31:03,237 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:31:03,238 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver Metrics about HBase RegionObservers 2023-07-22T12:31:03,238 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-07-22T12:31:03,238 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2023-07-22T12:31:03,238 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2023-07-22T12:31:03,238 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2023-07-22T12:31:03,257 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:31:03,381 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:31:03,670 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices to GuiceManagedComponentProvider with the scope "PerRequest" 2023-07-22T12:31:03,698 INFO [Thread-249 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:33785 2023-07-22T12:31:04,144 WARN [Listener at jenkins-hbase3.apache.org/42839 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-07-22T12:31:04,221 WARN [Listener at jenkins-hbase3.apache.org/42839 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-07-22T12:31:04,222 INFO [Listener at jenkins-hbase3.apache.org/42839 {}] log.Slf4jLog(67): jetty-6.1.26 2023-07-22T12:31:04,227 INFO [Listener at jenkins-hbase3.apache.org/42839 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/cluster to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_45579_cluster____nhe7xp/webapp 2023-07-22T12:31:04,306 INFO [Listener at jenkins-hbase3.apache.org/42839 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver as a provider class 2023-07-22T12:31:04,307 INFO [Listener at jenkins-hbase3.apache.org/42839 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices as a root resource class 2023-07-22T12:31:04,307 INFO [Listener at jenkins-hbase3.apache.org/42839 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-07-22T12:31:04,307 INFO [Listener at jenkins-hbase3.apache.org/42839 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-07-22T12:31:04,315 INFO [Listener at jenkins-hbase3.apache.org/42839 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:31:04,527 INFO [Listener at jenkins-hbase3.apache.org/42839 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:31:04,627 INFO [Listener at jenkins-hbase3.apache.org/42839 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:31:04,636 INFO [Listener at jenkins-hbase3.apache.org/42839 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:45579 2023-07-22T12:31:04,887 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-07-22T12:31:04,948 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2023-07-22T12:31:04,950 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2023-07-22T12:31:05,069 WARN [Listener at jenkins-hbase3.apache.org/38889 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-07-22T12:31:05,091 WARN [Listener at jenkins-hbase3.apache.org/38889 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-07-22T12:31:05,092 INFO [Listener at jenkins-hbase3.apache.org/38889 {}] log.Slf4jLog(67): jetty-6.1.26 2023-07-22T12:31:05,098 INFO [Listener at jenkins-hbase3.apache.org/38889 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_38629_node____.uf2xk9/webapp 2023-07-22T12:31:05,149 INFO [Listener at jenkins-hbase3.apache.org/38889 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-07-22T12:31:05,150 INFO [Listener at jenkins-hbase3.apache.org/38889 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-07-22T12:31:05,150 INFO [Listener at jenkins-hbase3.apache.org/38889 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-07-22T12:31:05,150 INFO [Listener at jenkins-hbase3.apache.org/38889 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-07-22T12:31:05,158 INFO [Listener at jenkins-hbase3.apache.org/38889 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:31:05,184 INFO [Listener at jenkins-hbase3.apache.org/38889 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:31:05,272 INFO [Listener at jenkins-hbase3.apache.org/38889 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:31:05,282 INFO [Listener at jenkins-hbase3.apache.org/38889 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:38629 2023-07-22T12:31:05,423 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-07-22T12:31:05,429 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-07-22T12:31:05,430 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] log.Slf4jLog(67): jetty-6.1.26 2023-07-22T12:31:05,436 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_42449_node____4uu5ju/webapp 2023-07-22T12:31:05,486 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-07-22T12:31:05,486 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-07-22T12:31:05,486 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-07-22T12:31:05,486 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-07-22T12:31:05,493 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:31:05,507 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:31:05,577 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:31:05,585 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:42449 2023-07-22T12:31:05,625 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.HBaseTestingUtility(2821): Mini mapreduce cluster started 2023-07-22T12:31:05,625 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] backup.TestBackupBase(301): ROOTDIR hdfs://localhost:34745/backupUT 2023-07-22T12:31:05,638 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:31:05,642 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:45764, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:31:05,653 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'ns1'} 2023-07-22T12:31:05,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=9, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns1 2023-07-22T12:31:05,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=9 2023-07-22T12:31:05,681 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-07-22T12:31:05,687 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=9, state=SUCCESS; CreateNamespaceProcedure, namespace=ns1 in 32 msec 2023-07-22T12:31:05,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=9 2023-07-22T12:31:05,776 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'ns2'} 2023-07-22T12:31:05,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=10, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns2 2023-07-22T12:31:05,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=10 2023-07-22T12:31:05,791 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-07-22T12:31:05,798 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=10, state=SUCCESS; CreateNamespaceProcedure, namespace=ns2 in 19 msec 2023-07-22T12:31:05,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=10 2023-07-22T12:31:05,886 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'ns3'} 2023-07-22T12:31:05,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=11, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns3 2023-07-22T12:31:05,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=11 2023-07-22T12:31:05,898 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-07-22T12:31:05,902 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=11, state=SUCCESS; CreateNamespaceProcedure, namespace=ns3 in 14 msec 2023-07-22T12:31:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=11 2023-07-22T12:31:05,995 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'ns4'} 2023-07-22T12:31:05,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=12, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns4 2023-07-22T12:31:06,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=12 2023-07-22T12:31:06,008 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-07-22T12:31:06,013 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=12, state=SUCCESS; CreateNamespaceProcedure, namespace=ns4 in 16 msec 2023-07-22T12:31:06,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=12 2023-07-22T12:31:06,115 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'test-1690029065625', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:31:06,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=test-1690029065625 2023-07-22T12:31:06,119 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=test-1690029065625 execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:31:06,119 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "test-1690029065625" procId is: 13 2023-07-22T12:31:06,119 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:31:06,120 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=test-1690029065625 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:31:06,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-07-22T12:31:06,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-07-22T12:31:06,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-07-22T12:31:06,554 INFO [RegionOpenAndInit-test-1690029065625-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 1a5339f2a886ed4b01c0668fe0d77c7f, NAME => 'test-1690029065625,,1690029066110.1a5339f2a886ed4b01c0668fe0d77c7f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1690029065625', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947 2023-07-22T12:31:06,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-07-22T12:31:06,970 DEBUG [RegionOpenAndInit-test-1690029065625-pool-0 {}] regionserver.HRegion(888): Instantiated test-1690029065625,,1690029066110.1a5339f2a886ed4b01c0668fe0d77c7f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:31:06,970 DEBUG [RegionOpenAndInit-test-1690029065625-pool-0 {}] regionserver.HRegion(1668): Closing 1a5339f2a886ed4b01c0668fe0d77c7f, disabling compactions & flushes 2023-07-22T12:31:06,970 INFO [RegionOpenAndInit-test-1690029065625-pool-0 {}] regionserver.HRegion(1690): Closing region test-1690029065625,,1690029066110.1a5339f2a886ed4b01c0668fe0d77c7f. 2023-07-22T12:31:06,970 DEBUG [RegionOpenAndInit-test-1690029065625-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1690029065625,,1690029066110.1a5339f2a886ed4b01c0668fe0d77c7f. 2023-07-22T12:31:06,970 DEBUG [RegionOpenAndInit-test-1690029065625-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on test-1690029065625,,1690029066110.1a5339f2a886ed4b01c0668fe0d77c7f. after waiting 0 ms 2023-07-22T12:31:06,970 DEBUG [RegionOpenAndInit-test-1690029065625-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region test-1690029065625,,1690029066110.1a5339f2a886ed4b01c0668fe0d77c7f. 2023-07-22T12:31:06,970 INFO [RegionOpenAndInit-test-1690029065625-pool-0 {}] regionserver.HRegion(1909): Closed test-1690029065625,,1690029066110.1a5339f2a886ed4b01c0668fe0d77c7f. 2023-07-22T12:31:06,970 DEBUG [RegionOpenAndInit-test-1690029065625-pool-0 {}] regionserver.HRegion(1622): Region close journal for 1a5339f2a886ed4b01c0668fe0d77c7f: 2023-07-22T12:31:06,972 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=test-1690029065625 execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:31:06,972 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"test-1690029065625,,1690029066110.1a5339f2a886ed4b01c0668fe0d77c7f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1690029066972"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690029066972"}]},"ts":"1690029066972"} 2023-07-22T12:31:06,974 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:31:06,976 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=test-1690029065625 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:31:06,976 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1690029065625","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029066976"}]},"ts":"1690029066976"} 2023-07-22T12:31:06,978 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1690029065625, state=ENABLING in hbase:meta 2023-07-22T12:31:06,983 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1690029065625, region=1a5339f2a886ed4b01c0668fe0d77c7f, ASSIGN}] 2023-07-22T12:31:06,985 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1690029065625, region=1a5339f2a886ed4b01c0668fe0d77c7f, ASSIGN 2023-07-22T12:31:06,986 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=test-1690029065625, region=1a5339f2a886ed4b01c0668fe0d77c7f, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,43639,1690029056327; forceNewPlan=false, retain=false 2023-07-22T12:31:07,138 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=1a5339f2a886ed4b01c0668fe0d77c7f, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:07,141 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE; OpenRegionProcedure 1a5339f2a886ed4b01c0668fe0d77c7f, server=jenkins-hbase3.apache.org,43639,1690029056327}] 2023-07-22T12:31:07,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-07-22T12:31:07,302 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(130): Open test-1690029065625,,1690029066110.1a5339f2a886ed4b01c0668fe0d77c7f. 2023-07-22T12:31:07,302 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7254): Opening region: {ENCODED => 1a5339f2a886ed4b01c0668fe0d77c7f, NAME => 'test-1690029065625,,1690029066110.1a5339f2a886ed4b01c0668fe0d77c7f.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:31:07,302 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:31:07,303 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1690029065625 1a5339f2a886ed4b01c0668fe0d77c7f 2023-07-22T12:31:07,303 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(888): Instantiated test-1690029065625,,1690029066110.1a5339f2a886ed4b01c0668fe0d77c7f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:31:07,303 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7296): checking encryption for 1a5339f2a886ed4b01c0668fe0d77c7f 2023-07-22T12:31:07,303 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7299): checking classloading for 1a5339f2a886ed4b01c0668fe0d77c7f 2023-07-22T12:31:07,305 INFO [StoreOpener-1a5339f2a886ed4b01c0668fe0d77c7f-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 1a5339f2a886ed4b01c0668fe0d77c7f 2023-07-22T12:31:07,307 INFO [StoreOpener-1a5339f2a886ed4b01c0668fe0d77c7f-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a5339f2a886ed4b01c0668fe0d77c7f columnFamilyName f 2023-07-22T12:31:07,307 DEBUG [StoreOpener-1a5339f2a886ed4b01c0668fe0d77c7f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:31:07,308 INFO [StoreOpener-1a5339f2a886ed4b01c0668fe0d77c7f-1 {}] regionserver.HStore(324): Store=1a5339f2a886ed4b01c0668fe0d77c7f/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:31:07,309 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/1a5339f2a886ed4b01c0668fe0d77c7f 2023-07-22T12:31:07,310 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/1a5339f2a886ed4b01c0668fe0d77c7f 2023-07-22T12:31:07,313 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1076): writing seq id for 1a5339f2a886ed4b01c0668fe0d77c7f 2023-07-22T12:31:07,319 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/1a5339f2a886ed4b01c0668fe0d77c7f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:31:07,319 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): Opened 1a5339f2a886ed4b01c0668fe0d77c7f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10807830080, jitterRate=0.0065576136112213135}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-07-22T12:31:07,320 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(992): Region open journal for 1a5339f2a886ed4b01c0668fe0d77c7f: 2023-07-22T12:31:07,321 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2552): Post open deploy tasks for test-1690029065625,,1690029066110.1a5339f2a886ed4b01c0668fe0d77c7f., pid=15, masterSystemTime=1690029067294 2023-07-22T12:31:07,324 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2579): Finished post open deploy task for test-1690029065625,,1690029066110.1a5339f2a886ed4b01c0668fe0d77c7f. 2023-07-22T12:31:07,324 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(158): Opened test-1690029065625,,1690029066110.1a5339f2a886ed4b01c0668fe0d77c7f. 2023-07-22T12:31:07,325 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=1a5339f2a886ed4b01c0668fe0d77c7f, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:07,330 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=15, resume processing ppid=14 2023-07-22T12:31:07,330 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=15, ppid=14, state=SUCCESS; OpenRegionProcedure 1a5339f2a886ed4b01c0668fe0d77c7f, server=jenkins-hbase3.apache.org,43639,1690029056327 in 186 msec 2023-07-22T12:31:07,333 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=14, resume processing ppid=13 2023-07-22T12:31:07,333 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=14, ppid=13, state=SUCCESS; TransitRegionStateProcedure table=test-1690029065625, region=1a5339f2a886ed4b01c0668fe0d77c7f, ASSIGN in 348 msec 2023-07-22T12:31:07,334 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=test-1690029065625 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:31:07,334 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1690029065625","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029067334"}]},"ts":"1690029067334"} 2023-07-22T12:31:07,336 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1690029065625, state=ENABLED in hbase:meta 2023-07-22T12:31:07,339 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=test-1690029065625 execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:31:07,341 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=13, state=SUCCESS; CreateTableProcedure table=test-1690029065625 in 1.2240 sec 2023-07-22T12:31:08,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-07-22T12:31:08,236 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: default:test-1690029065625, procId: 13 completed 2023-07-22T12:31:08,236 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x38750ad2 to 127.0.0.1:62657 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:31:08,248 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4395358, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:31:08,250 DEBUG [hconnection-0x26ae111c-metaLookup-shared--pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:31:08,253 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:36672, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:31:08,258 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HRegion(8210): writing data to region test-1690029065625,,1690029066110.1a5339f2a886ed4b01c0668fe0d77c7f. with WAL disabled. Data may be lost in the event of a crash. 2023-07-22T12:31:08,374 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'ns2:test-16900290656251', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:31:08,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns2:test-16900290656251 2023-07-22T12:31:08,378 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16900290656251 execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:31:08,378 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:31:08,378 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns2" qualifier: "test-16900290656251" procId is: 16 2023-07-22T12:31:08,380 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns2:test-16900290656251 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:31:08,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-07-22T12:31:08,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-07-22T12:31:08,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-07-22T12:31:08,797 INFO [RegionOpenAndInit-ns2:test-16900290656251-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 0112d97ae6696b92341dfd7ff8824a0a, NAME => 'ns2:test-16900290656251,,1690029068374.0112d97ae6696b92341dfd7ff8824a0a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-16900290656251', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947 2023-07-22T12:31:08,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-07-22T12:31:09,212 DEBUG [RegionOpenAndInit-ns2:test-16900290656251-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:test-16900290656251,,1690029068374.0112d97ae6696b92341dfd7ff8824a0a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:31:09,212 DEBUG [RegionOpenAndInit-ns2:test-16900290656251-pool-0 {}] regionserver.HRegion(1668): Closing 0112d97ae6696b92341dfd7ff8824a0a, disabling compactions & flushes 2023-07-22T12:31:09,212 INFO [RegionOpenAndInit-ns2:test-16900290656251-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:test-16900290656251,,1690029068374.0112d97ae6696b92341dfd7ff8824a0a. 2023-07-22T12:31:09,212 DEBUG [RegionOpenAndInit-ns2:test-16900290656251-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16900290656251,,1690029068374.0112d97ae6696b92341dfd7ff8824a0a. 2023-07-22T12:31:09,212 DEBUG [RegionOpenAndInit-ns2:test-16900290656251-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16900290656251,,1690029068374.0112d97ae6696b92341dfd7ff8824a0a. after waiting 0 ms 2023-07-22T12:31:09,212 DEBUG [RegionOpenAndInit-ns2:test-16900290656251-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16900290656251,,1690029068374.0112d97ae6696b92341dfd7ff8824a0a. 2023-07-22T12:31:09,212 INFO [RegionOpenAndInit-ns2:test-16900290656251-pool-0 {}] regionserver.HRegion(1909): Closed ns2:test-16900290656251,,1690029068374.0112d97ae6696b92341dfd7ff8824a0a. 2023-07-22T12:31:09,212 DEBUG [RegionOpenAndInit-ns2:test-16900290656251-pool-0 {}] regionserver.HRegion(1622): Region close journal for 0112d97ae6696b92341dfd7ff8824a0a: 2023-07-22T12:31:09,214 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns2:test-16900290656251 execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:31:09,214 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:test-16900290656251,,1690029068374.0112d97ae6696b92341dfd7ff8824a0a.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1690029069214"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690029069214"}]},"ts":"1690029069214"} 2023-07-22T12:31:09,217 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:31:09,218 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns2:test-16900290656251 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:31:09,218 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16900290656251","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029069218"}]},"ts":"1690029069218"} 2023-07-22T12:31:09,220 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16900290656251, state=ENABLING in hbase:meta 2023-07-22T12:31:09,225 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16900290656251, region=0112d97ae6696b92341dfd7ff8824a0a, ASSIGN}] 2023-07-22T12:31:09,227 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16900290656251, region=0112d97ae6696b92341dfd7ff8824a0a, ASSIGN 2023-07-22T12:31:09,228 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:test-16900290656251, region=0112d97ae6696b92341dfd7ff8824a0a, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,43639,1690029056327; forceNewPlan=false, retain=false 2023-07-22T12:31:09,380 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=0112d97ae6696b92341dfd7ff8824a0a, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:09,383 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE; OpenRegionProcedure 0112d97ae6696b92341dfd7ff8824a0a, server=jenkins-hbase3.apache.org,43639,1690029056327}] 2023-07-22T12:31:09,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-07-22T12:31:09,542 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(130): Open ns2:test-16900290656251,,1690029068374.0112d97ae6696b92341dfd7ff8824a0a. 2023-07-22T12:31:09,543 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7254): Opening region: {ENCODED => 0112d97ae6696b92341dfd7ff8824a0a, NAME => 'ns2:test-16900290656251,,1690029068374.0112d97ae6696b92341dfd7ff8824a0a.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:31:09,543 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:31:09,543 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16900290656251 0112d97ae6696b92341dfd7ff8824a0a 2023-07-22T12:31:09,543 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(888): Instantiated ns2:test-16900290656251,,1690029068374.0112d97ae6696b92341dfd7ff8824a0a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:31:09,543 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7296): checking encryption for 0112d97ae6696b92341dfd7ff8824a0a 2023-07-22T12:31:09,543 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7299): checking classloading for 0112d97ae6696b92341dfd7ff8824a0a 2023-07-22T12:31:09,545 INFO [StoreOpener-0112d97ae6696b92341dfd7ff8824a0a-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 0112d97ae6696b92341dfd7ff8824a0a 2023-07-22T12:31:09,547 INFO [StoreOpener-0112d97ae6696b92341dfd7ff8824a0a-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0112d97ae6696b92341dfd7ff8824a0a columnFamilyName f 2023-07-22T12:31:09,547 DEBUG [StoreOpener-0112d97ae6696b92341dfd7ff8824a0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:31:09,548 INFO [StoreOpener-0112d97ae6696b92341dfd7ff8824a0a-1 {}] regionserver.HStore(324): Store=0112d97ae6696b92341dfd7ff8824a0a/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:31:09,549 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/0112d97ae6696b92341dfd7ff8824a0a 2023-07-22T12:31:09,549 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/0112d97ae6696b92341dfd7ff8824a0a 2023-07-22T12:31:09,552 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1076): writing seq id for 0112d97ae6696b92341dfd7ff8824a0a 2023-07-22T12:31:09,559 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/0112d97ae6696b92341dfd7ff8824a0a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:31:09,559 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): Opened 0112d97ae6696b92341dfd7ff8824a0a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11192772480, jitterRate=0.04240816831588745}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-07-22T12:31:09,560 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(992): Region open journal for 0112d97ae6696b92341dfd7ff8824a0a: 2023-07-22T12:31:09,561 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2552): Post open deploy tasks for ns2:test-16900290656251,,1690029068374.0112d97ae6696b92341dfd7ff8824a0a., pid=18, masterSystemTime=1690029069535 2023-07-22T12:31:09,563 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2579): Finished post open deploy task for ns2:test-16900290656251,,1690029068374.0112d97ae6696b92341dfd7ff8824a0a. 2023-07-22T12:31:09,563 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(158): Opened ns2:test-16900290656251,,1690029068374.0112d97ae6696b92341dfd7ff8824a0a. 2023-07-22T12:31:09,564 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=0112d97ae6696b92341dfd7ff8824a0a, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:09,569 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=18, resume processing ppid=17 2023-07-22T12:31:09,569 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=18, ppid=17, state=SUCCESS; OpenRegionProcedure 0112d97ae6696b92341dfd7ff8824a0a, server=jenkins-hbase3.apache.org,43639,1690029056327 in 183 msec 2023-07-22T12:31:09,572 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=17, resume processing ppid=16 2023-07-22T12:31:09,572 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=17, ppid=16, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-16900290656251, region=0112d97ae6696b92341dfd7ff8824a0a, ASSIGN in 344 msec 2023-07-22T12:31:09,573 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns2:test-16900290656251 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:31:09,573 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16900290656251","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029069573"}]},"ts":"1690029069573"} 2023-07-22T12:31:09,575 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16900290656251, state=ENABLED in hbase:meta 2023-07-22T12:31:09,578 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16900290656251 execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:31:09,580 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=16, state=SUCCESS; CreateTableProcedure table=ns2:test-16900290656251 in 1.2030 sec 2023-07-22T12:31:10,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-07-22T12:31:10,488 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: ns2:test-16900290656251, procId: 16 completed 2023-07-22T12:31:10,493 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HRegion(8210): writing data to region ns2:test-16900290656251,,1690029068374.0112d97ae6696b92341dfd7ff8824a0a. with WAL disabled. Data may be lost in the event of a crash. 2023-07-22T12:31:10,575 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'ns3:test-16900290656252', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:31:10,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns3:test-16900290656252 2023-07-22T12:31:10,580 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16900290656252 execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:31:10,580 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:31:10,580 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns3" qualifier: "test-16900290656252" procId is: 19 2023-07-22T12:31:10,581 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns3:test-16900290656252 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:31:10,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-07-22T12:31:10,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-07-22T12:31:10,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-07-22T12:31:10,955 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test-1690029065625' 2023-07-22T12:31:10,955 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:test-16900290656251' 2023-07-22T12:31:11,008 INFO [RegionOpenAndInit-ns3:test-16900290656252-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 335c94783192a14614120d955d2aa716, NAME => 'ns3:test-16900290656252,,1690029070575.335c94783192a14614120d955d2aa716.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-16900290656252', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947 2023-07-22T12:31:11,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-07-22T12:31:11,425 DEBUG [RegionOpenAndInit-ns3:test-16900290656252-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:test-16900290656252,,1690029070575.335c94783192a14614120d955d2aa716.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:31:11,425 DEBUG [RegionOpenAndInit-ns3:test-16900290656252-pool-0 {}] regionserver.HRegion(1668): Closing 335c94783192a14614120d955d2aa716, disabling compactions & flushes 2023-07-22T12:31:11,425 INFO [RegionOpenAndInit-ns3:test-16900290656252-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:test-16900290656252,,1690029070575.335c94783192a14614120d955d2aa716. 2023-07-22T12:31:11,425 DEBUG [RegionOpenAndInit-ns3:test-16900290656252-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16900290656252,,1690029070575.335c94783192a14614120d955d2aa716. 2023-07-22T12:31:11,425 DEBUG [RegionOpenAndInit-ns3:test-16900290656252-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16900290656252,,1690029070575.335c94783192a14614120d955d2aa716. after waiting 0 ms 2023-07-22T12:31:11,425 DEBUG [RegionOpenAndInit-ns3:test-16900290656252-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16900290656252,,1690029070575.335c94783192a14614120d955d2aa716. 2023-07-22T12:31:11,425 INFO [RegionOpenAndInit-ns3:test-16900290656252-pool-0 {}] regionserver.HRegion(1909): Closed ns3:test-16900290656252,,1690029070575.335c94783192a14614120d955d2aa716. 2023-07-22T12:31:11,425 DEBUG [RegionOpenAndInit-ns3:test-16900290656252-pool-0 {}] regionserver.HRegion(1622): Region close journal for 335c94783192a14614120d955d2aa716: 2023-07-22T12:31:11,427 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns3:test-16900290656252 execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:31:11,427 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:test-16900290656252,,1690029070575.335c94783192a14614120d955d2aa716.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1690029071427"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690029071427"}]},"ts":"1690029071427"} 2023-07-22T12:31:11,429 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:31:11,430 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns3:test-16900290656252 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:31:11,431 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16900290656252","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029071430"}]},"ts":"1690029071430"} 2023-07-22T12:31:11,432 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16900290656252, state=ENABLING in hbase:meta 2023-07-22T12:31:11,441 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16900290656252, region=335c94783192a14614120d955d2aa716, ASSIGN}] 2023-07-22T12:31:11,443 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16900290656252, region=335c94783192a14614120d955d2aa716, ASSIGN 2023-07-22T12:31:11,444 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:test-16900290656252, region=335c94783192a14614120d955d2aa716, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,43639,1690029056327; forceNewPlan=false, retain=false 2023-07-22T12:31:11,595 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=335c94783192a14614120d955d2aa716, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:11,598 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE; OpenRegionProcedure 335c94783192a14614120d955d2aa716, server=jenkins-hbase3.apache.org,43639,1690029056327}] 2023-07-22T12:31:11,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-07-22T12:31:11,755 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(130): Open ns3:test-16900290656252,,1690029070575.335c94783192a14614120d955d2aa716. 2023-07-22T12:31:11,755 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7254): Opening region: {ENCODED => 335c94783192a14614120d955d2aa716, NAME => 'ns3:test-16900290656252,,1690029070575.335c94783192a14614120d955d2aa716.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:31:11,756 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:31:11,756 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16900290656252 335c94783192a14614120d955d2aa716 2023-07-22T12:31:11,756 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(888): Instantiated ns3:test-16900290656252,,1690029070575.335c94783192a14614120d955d2aa716.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:31:11,756 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7296): checking encryption for 335c94783192a14614120d955d2aa716 2023-07-22T12:31:11,756 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7299): checking classloading for 335c94783192a14614120d955d2aa716 2023-07-22T12:31:11,758 INFO [StoreOpener-335c94783192a14614120d955d2aa716-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 335c94783192a14614120d955d2aa716 2023-07-22T12:31:11,760 INFO [StoreOpener-335c94783192a14614120d955d2aa716-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 335c94783192a14614120d955d2aa716 columnFamilyName f 2023-07-22T12:31:11,760 DEBUG [StoreOpener-335c94783192a14614120d955d2aa716-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:31:11,760 INFO [StoreOpener-335c94783192a14614120d955d2aa716-1 {}] regionserver.HStore(324): Store=335c94783192a14614120d955d2aa716/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:31:11,761 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns3/test-16900290656252/335c94783192a14614120d955d2aa716 2023-07-22T12:31:11,762 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns3/test-16900290656252/335c94783192a14614120d955d2aa716 2023-07-22T12:31:11,765 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1076): writing seq id for 335c94783192a14614120d955d2aa716 2023-07-22T12:31:11,768 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns3/test-16900290656252/335c94783192a14614120d955d2aa716/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:31:11,768 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): Opened 335c94783192a14614120d955d2aa716; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10154080320, jitterRate=-0.054327577352523804}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-07-22T12:31:11,769 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(992): Region open journal for 335c94783192a14614120d955d2aa716: 2023-07-22T12:31:11,770 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2552): Post open deploy tasks for ns3:test-16900290656252,,1690029070575.335c94783192a14614120d955d2aa716., pid=21, masterSystemTime=1690029071750 2023-07-22T12:31:11,773 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2579): Finished post open deploy task for ns3:test-16900290656252,,1690029070575.335c94783192a14614120d955d2aa716. 2023-07-22T12:31:11,773 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(158): Opened ns3:test-16900290656252,,1690029070575.335c94783192a14614120d955d2aa716. 2023-07-22T12:31:11,773 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=335c94783192a14614120d955d2aa716, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:11,778 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=21, resume processing ppid=20 2023-07-22T12:31:11,778 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=21, ppid=20, state=SUCCESS; OpenRegionProcedure 335c94783192a14614120d955d2aa716, server=jenkins-hbase3.apache.org,43639,1690029056327 in 177 msec 2023-07-22T12:31:11,780 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=20, resume processing ppid=19 2023-07-22T12:31:11,780 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=20, ppid=19, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-16900290656252, region=335c94783192a14614120d955d2aa716, ASSIGN in 337 msec 2023-07-22T12:31:11,781 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns3:test-16900290656252 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:31:11,781 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16900290656252","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029071781"}]},"ts":"1690029071781"} 2023-07-22T12:31:11,783 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16900290656252, state=ENABLED in hbase:meta 2023-07-22T12:31:11,785 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16900290656252 execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:31:11,787 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=19, state=SUCCESS; CreateTableProcedure table=ns3:test-16900290656252 in 1.2090 sec 2023-07-22T12:31:12,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-07-22T12:31:12,691 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: ns3:test-16900290656252, procId: 19 completed 2023-07-22T12:31:12,691 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns3:test-16900290656252 get assigned. Timeout = 60000ms 2023-07-22T12:31:12,693 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-07-22T12:31:12,698 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.HBaseTestingUtility(3581): All regions for table ns3:test-16900290656252 assigned to meta. Checking AM states. 2023-07-22T12:31:12,699 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-07-22T12:31:12,699 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.HBaseTestingUtility(3601): All regions for table ns3:test-16900290656252 assigned. 2023-07-22T12:31:12,701 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'ns4:test-16900290656253', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:31:12,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns4:test-16900290656253 2023-07-22T12:31:12,706 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16900290656253 execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:31:12,706 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:31:12,706 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns4" qualifier: "test-16900290656253" procId is: 22 2023-07-22T12:31:12,707 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns4:test-16900290656253 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:31:12,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-07-22T12:31:12,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-07-22T12:31:12,811 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-07-22T12:31:13,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-07-22T12:31:13,127 INFO [RegionOpenAndInit-ns4:test-16900290656253-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 54af1049fa09c4790d7061c581618a59, NAME => 'ns4:test-16900290656253,,1690029072701.54af1049fa09c4790d7061c581618a59.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns4:test-16900290656253', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947 2023-07-22T12:31:13,225 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-16900290656252 2023-07-22T12:31:13,225 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns3_table_test-16900290656252 Metrics about Tables on a single HBase RegionServer 2023-07-22T12:31:13,226 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1690029065625 2023-07-22T12:31:13,226 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test-1690029065625 Metrics about Tables on a single HBase RegionServer 2023-07-22T12:31:13,227 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-16900290656251 2023-07-22T12:31:13,227 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_test-16900290656251 Metrics about Tables on a single HBase RegionServer 2023-07-22T12:31:13,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-07-22T12:31:13,541 DEBUG [RegionOpenAndInit-ns4:test-16900290656253-pool-0 {}] regionserver.HRegion(888): Instantiated ns4:test-16900290656253,,1690029072701.54af1049fa09c4790d7061c581618a59.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:31:13,541 DEBUG [RegionOpenAndInit-ns4:test-16900290656253-pool-0 {}] regionserver.HRegion(1668): Closing 54af1049fa09c4790d7061c581618a59, disabling compactions & flushes 2023-07-22T12:31:13,541 INFO [RegionOpenAndInit-ns4:test-16900290656253-pool-0 {}] regionserver.HRegion(1690): Closing region ns4:test-16900290656253,,1690029072701.54af1049fa09c4790d7061c581618a59. 2023-07-22T12:31:13,541 DEBUG [RegionOpenAndInit-ns4:test-16900290656253-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16900290656253,,1690029072701.54af1049fa09c4790d7061c581618a59. 2023-07-22T12:31:13,542 DEBUG [RegionOpenAndInit-ns4:test-16900290656253-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16900290656253,,1690029072701.54af1049fa09c4790d7061c581618a59. after waiting 0 ms 2023-07-22T12:31:13,542 DEBUG [RegionOpenAndInit-ns4:test-16900290656253-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16900290656253,,1690029072701.54af1049fa09c4790d7061c581618a59. 2023-07-22T12:31:13,542 INFO [RegionOpenAndInit-ns4:test-16900290656253-pool-0 {}] regionserver.HRegion(1909): Closed ns4:test-16900290656253,,1690029072701.54af1049fa09c4790d7061c581618a59. 2023-07-22T12:31:13,542 DEBUG [RegionOpenAndInit-ns4:test-16900290656253-pool-0 {}] regionserver.HRegion(1622): Region close journal for 54af1049fa09c4790d7061c581618a59: 2023-07-22T12:31:13,543 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns4:test-16900290656253 execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:31:13,544 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns4:test-16900290656253,,1690029072701.54af1049fa09c4790d7061c581618a59.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1690029073543"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690029073543"}]},"ts":"1690029073543"} 2023-07-22T12:31:13,546 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:31:13,547 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns4:test-16900290656253 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:31:13,547 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16900290656253","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029073547"}]},"ts":"1690029073547"} 2023-07-22T12:31:13,549 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16900290656253, state=ENABLING in hbase:meta 2023-07-22T12:31:13,554 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16900290656253, region=54af1049fa09c4790d7061c581618a59, ASSIGN}] 2023-07-22T12:31:13,556 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16900290656253, region=54af1049fa09c4790d7061c581618a59, ASSIGN 2023-07-22T12:31:13,557 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns4:test-16900290656253, region=54af1049fa09c4790d7061c581618a59, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,43639,1690029056327; forceNewPlan=false, retain=false 2023-07-22T12:31:13,708 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=54af1049fa09c4790d7061c581618a59, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:13,711 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE; OpenRegionProcedure 54af1049fa09c4790d7061c581618a59, server=jenkins-hbase3.apache.org,43639,1690029056327}] 2023-07-22T12:31:13,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-07-22T12:31:13,868 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(130): Open ns4:test-16900290656253,,1690029072701.54af1049fa09c4790d7061c581618a59. 2023-07-22T12:31:13,868 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7254): Opening region: {ENCODED => 54af1049fa09c4790d7061c581618a59, NAME => 'ns4:test-16900290656253,,1690029072701.54af1049fa09c4790d7061c581618a59.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:31:13,868 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:31:13,869 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16900290656253 54af1049fa09c4790d7061c581618a59 2023-07-22T12:31:13,869 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(888): Instantiated ns4:test-16900290656253,,1690029072701.54af1049fa09c4790d7061c581618a59.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:31:13,869 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7296): checking encryption for 54af1049fa09c4790d7061c581618a59 2023-07-22T12:31:13,869 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7299): checking classloading for 54af1049fa09c4790d7061c581618a59 2023-07-22T12:31:13,871 INFO [StoreOpener-54af1049fa09c4790d7061c581618a59-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 54af1049fa09c4790d7061c581618a59 2023-07-22T12:31:13,873 INFO [StoreOpener-54af1049fa09c4790d7061c581618a59-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 54af1049fa09c4790d7061c581618a59 columnFamilyName f 2023-07-22T12:31:13,873 DEBUG [StoreOpener-54af1049fa09c4790d7061c581618a59-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:31:13,874 INFO [StoreOpener-54af1049fa09c4790d7061c581618a59-1 {}] regionserver.HStore(324): Store=54af1049fa09c4790d7061c581618a59/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:31:13,875 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns4/test-16900290656253/54af1049fa09c4790d7061c581618a59 2023-07-22T12:31:13,875 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns4/test-16900290656253/54af1049fa09c4790d7061c581618a59 2023-07-22T12:31:13,878 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1076): writing seq id for 54af1049fa09c4790d7061c581618a59 2023-07-22T12:31:13,881 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns4/test-16900290656253/54af1049fa09c4790d7061c581618a59/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:31:13,881 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): Opened 54af1049fa09c4790d7061c581618a59; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11410021920, jitterRate=0.06264109909534454}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-07-22T12:31:13,882 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(992): Region open journal for 54af1049fa09c4790d7061c581618a59: 2023-07-22T12:31:13,883 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2552): Post open deploy tasks for ns4:test-16900290656253,,1690029072701.54af1049fa09c4790d7061c581618a59., pid=24, masterSystemTime=1690029073863 2023-07-22T12:31:13,885 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2579): Finished post open deploy task for ns4:test-16900290656253,,1690029072701.54af1049fa09c4790d7061c581618a59. 2023-07-22T12:31:13,885 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(158): Opened ns4:test-16900290656253,,1690029072701.54af1049fa09c4790d7061c581618a59. 2023-07-22T12:31:13,886 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=54af1049fa09c4790d7061c581618a59, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:13,891 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=24, resume processing ppid=23 2023-07-22T12:31:13,891 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=24, ppid=23, state=SUCCESS; OpenRegionProcedure 54af1049fa09c4790d7061c581618a59, server=jenkins-hbase3.apache.org,43639,1690029056327 in 178 msec 2023-07-22T12:31:13,894 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=23, resume processing ppid=22 2023-07-22T12:31:13,894 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=23, ppid=22, state=SUCCESS; TransitRegionStateProcedure table=ns4:test-16900290656253, region=54af1049fa09c4790d7061c581618a59, ASSIGN in 337 msec 2023-07-22T12:31:13,895 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns4:test-16900290656253 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:31:13,895 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16900290656253","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029073895"}]},"ts":"1690029073895"} 2023-07-22T12:31:13,897 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16900290656253, state=ENABLED in hbase:meta 2023-07-22T12:31:13,899 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16900290656253 execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:31:13,901 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=22, state=SUCCESS; CreateTableProcedure table=ns4:test-16900290656253 in 1.1970 sec 2023-07-22T12:31:14,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-07-22T12:31:14,816 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: ns4:test-16900290656253, procId: 22 completed 2023-07-22T12:31:14,816 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns4:test-16900290656253 get assigned. Timeout = 60000ms 2023-07-22T12:31:14,816 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-07-22T12:31:14,819 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.HBaseTestingUtility(3581): All regions for table ns4:test-16900290656253 assigned to meta. Checking AM states. 2023-07-22T12:31:14,820 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-07-22T12:31:14,820 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.HBaseTestingUtility(3601): All regions for table ns4:test-16900290656253 assigned. 2023-07-22T12:31:14,820 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x38750ad2 to 127.0.0.1:62657 2023-07-22T12:31:14,820 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:31:14,859 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreMultipleOverwrite Thread=582, OpenFileDescriptor=660, MaxFileDescriptor=60000, SystemLoadAverage=156, ProcessCount=169, AvailableMemoryMB=16702 2023-07-22T12:31:14,859 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(130): Thread=582 is superior to 500 2023-07-22T12:31:14,862 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] backup.TestFullRestore(216): create full backup image on multiple tables 2023-07-22T12:31:14,862 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x63fd3bc4 to 127.0.0.1:62657 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:31:14,868 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a682e51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:31:14,877 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:31:14,879 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:58146, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:31:14,891 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:31:14,892 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:39698, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:31:14,902 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'backup'} 2023-07-22T12:31:14,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=25, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=backup 2023-07-22T12:31:14,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=25 2023-07-22T12:31:14,916 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-07-22T12:31:14,920 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=25, state=SUCCESS; CreateNamespaceProcedure, namespace=backup in 17 msec 2023-07-22T12:31:15,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=25 2023-07-22T12:31:15,024 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'backup:system', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:31:15,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system 2023-07-22T12:31:15,028 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:31:15,028 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:31:15,028 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system" procId is: 26 2023-07-22T12:31:15,029 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:31:15,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=26 2023-07-22T12:31:15,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=26 2023-07-22T12:31:15,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=26 2023-07-22T12:31:15,447 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 697cc5ecf0962f5ab9f4126febf7ce33, NAME => 'backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947 2023-07-22T12:31:15,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=26 2023-07-22T12:31:15,859 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:31:15,859 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1668): Closing 697cc5ecf0962f5ab9f4126febf7ce33, disabling compactions & flushes 2023-07-22T12:31:15,859 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. 2023-07-22T12:31:15,859 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. 2023-07-22T12:31:15,859 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. after waiting 0 ms 2023-07-22T12:31:15,859 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. 2023-07-22T12:31:15,859 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1909): Closed backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. 2023-07-22T12:31:15,859 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1622): Region close journal for 697cc5ecf0962f5ab9f4126febf7ce33: 2023-07-22T12:31:15,861 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:31:15,861 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1690029075861"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690029075861"}]},"ts":"1690029075861"} 2023-07-22T12:31:15,863 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:31:15,864 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:31:15,864 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029075864"}]},"ts":"1690029075864"} 2023-07-22T12:31:15,866 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-07-22T12:31:15,870 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=697cc5ecf0962f5ab9f4126febf7ce33, ASSIGN}] 2023-07-22T12:31:15,872 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=697cc5ecf0962f5ab9f4126febf7ce33, ASSIGN 2023-07-22T12:31:15,873 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=697cc5ecf0962f5ab9f4126febf7ce33, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,43639,1690029056327; forceNewPlan=false, retain=false 2023-07-22T12:31:16,024 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=697cc5ecf0962f5ab9f4126febf7ce33, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:16,027 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE; OpenRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33, server=jenkins-hbase3.apache.org,43639,1690029056327}] 2023-07-22T12:31:16,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=26 2023-07-22T12:31:16,184 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(130): Open backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. 2023-07-22T12:31:16,184 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7254): Opening region: {ENCODED => 697cc5ecf0962f5ab9f4126febf7ce33, NAME => 'backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:31:16,185 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:31:16,185 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 697cc5ecf0962f5ab9f4126febf7ce33 2023-07-22T12:31:16,185 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(888): Instantiated backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:31:16,185 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7296): checking encryption for 697cc5ecf0962f5ab9f4126febf7ce33 2023-07-22T12:31:16,185 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7299): checking classloading for 697cc5ecf0962f5ab9f4126febf7ce33 2023-07-22T12:31:16,186 INFO [StoreOpener-697cc5ecf0962f5ab9f4126febf7ce33-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 697cc5ecf0962f5ab9f4126febf7ce33 2023-07-22T12:31:16,189 INFO [StoreOpener-697cc5ecf0962f5ab9f4126febf7ce33-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 697cc5ecf0962f5ab9f4126febf7ce33 columnFamilyName meta 2023-07-22T12:31:16,190 DEBUG [StoreOpener-697cc5ecf0962f5ab9f4126febf7ce33-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:31:16,191 INFO [StoreOpener-697cc5ecf0962f5ab9f4126febf7ce33-1 {}] regionserver.HStore(324): Store=697cc5ecf0962f5ab9f4126febf7ce33/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:31:16,191 INFO [StoreOpener-697cc5ecf0962f5ab9f4126febf7ce33-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 697cc5ecf0962f5ab9f4126febf7ce33 2023-07-22T12:31:16,192 INFO [StoreOpener-697cc5ecf0962f5ab9f4126febf7ce33-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 697cc5ecf0962f5ab9f4126febf7ce33 columnFamilyName session 2023-07-22T12:31:16,192 DEBUG [StoreOpener-697cc5ecf0962f5ab9f4126febf7ce33-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:31:16,193 INFO [StoreOpener-697cc5ecf0962f5ab9f4126febf7ce33-1 {}] regionserver.HStore(324): Store=697cc5ecf0962f5ab9f4126febf7ce33/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:31:16,194 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33 2023-07-22T12:31:16,194 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33 2023-07-22T12:31:16,196 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-07-22T12:31:16,197 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1076): writing seq id for 697cc5ecf0962f5ab9f4126febf7ce33 2023-07-22T12:31:16,200 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:31:16,201 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): Opened 697cc5ecf0962f5ab9f4126febf7ce33; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9547812320, jitterRate=-0.1107906848192215}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-07-22T12:31:16,201 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(992): Region open journal for 697cc5ecf0962f5ab9f4126febf7ce33: 2023-07-22T12:31:16,202 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2552): Post open deploy tasks for backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33., pid=28, masterSystemTime=1690029076179 2023-07-22T12:31:16,204 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2579): Finished post open deploy task for backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. 2023-07-22T12:31:16,204 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(158): Opened backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. 2023-07-22T12:31:16,205 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=697cc5ecf0962f5ab9f4126febf7ce33, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:16,208 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=28, resume processing ppid=27 2023-07-22T12:31:16,208 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=28, ppid=27, state=SUCCESS; OpenRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33, server=jenkins-hbase3.apache.org,43639,1690029056327 in 179 msec 2023-07-22T12:31:16,210 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=27, resume processing ppid=26 2023-07-22T12:31:16,211 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=27, ppid=26, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=697cc5ecf0962f5ab9f4126febf7ce33, ASSIGN in 338 msec 2023-07-22T12:31:16,211 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:31:16,212 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029076211"}]},"ts":"1690029076211"} 2023-07-22T12:31:16,213 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-07-22T12:31:16,216 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:31:16,218 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=26, state=SUCCESS; CreateTableProcedure table=backup:system in 1.1920 sec 2023-07-22T12:31:16,956 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns4:test-16900290656253' 2023-07-22T12:31:16,956 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-07-22T12:31:16,957 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-16900290656252' 2023-07-22T12:31:17,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=26 2023-07-22T12:31:17,137 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: backup:system, procId: 26 completed 2023-07-22T12:31:17,151 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'backup:system_bulk', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:31:17,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system_bulk 2023-07-22T12:31:17,155 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:31:17,155 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:31:17,155 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system_bulk" procId is: 29 2023-07-22T12:31:17,156 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:31:17,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-07-22T12:31:17,169 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => fc6cb6b064d3fb792c9e01da5e382760, NAME => 'backup:system_bulk,,1690029077151.fc6cb6b064d3fb792c9e01da5e382760.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947 2023-07-22T12:31:17,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-07-22T12:31:17,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-07-22T12:31:17,583 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1690029077151.fc6cb6b064d3fb792c9e01da5e382760.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:31:17,583 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1668): Closing fc6cb6b064d3fb792c9e01da5e382760, disabling compactions & flushes 2023-07-22T12:31:17,583 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1690029077151.fc6cb6b064d3fb792c9e01da5e382760. 2023-07-22T12:31:17,583 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1690029077151.fc6cb6b064d3fb792c9e01da5e382760. 2023-07-22T12:31:17,583 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1690029077151.fc6cb6b064d3fb792c9e01da5e382760. after waiting 0 ms 2023-07-22T12:31:17,583 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1690029077151.fc6cb6b064d3fb792c9e01da5e382760. 2023-07-22T12:31:17,583 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1909): Closed backup:system_bulk,,1690029077151.fc6cb6b064d3fb792c9e01da5e382760. 2023-07-22T12:31:17,583 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1622): Region close journal for fc6cb6b064d3fb792c9e01da5e382760: 2023-07-22T12:31:17,585 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:31:17,585 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system_bulk,,1690029077151.fc6cb6b064d3fb792c9e01da5e382760.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1690029077585"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690029077585"}]},"ts":"1690029077585"} 2023-07-22T12:31:17,587 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:31:17,588 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:31:17,588 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029077588"}]},"ts":"1690029077588"} 2023-07-22T12:31:17,589 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLING in hbase:meta 2023-07-22T12:31:17,594 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=fc6cb6b064d3fb792c9e01da5e382760, ASSIGN}] 2023-07-22T12:31:17,596 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=fc6cb6b064d3fb792c9e01da5e382760, ASSIGN 2023-07-22T12:31:17,597 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system_bulk, region=fc6cb6b064d3fb792c9e01da5e382760, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,43639,1690029056327; forceNewPlan=false, retain=false 2023-07-22T12:31:17,749 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=fc6cb6b064d3fb792c9e01da5e382760, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:17,752 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE; OpenRegionProcedure fc6cb6b064d3fb792c9e01da5e382760, server=jenkins-hbase3.apache.org,43639,1690029056327}] 2023-07-22T12:31:17,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-07-22T12:31:17,909 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(130): Open backup:system_bulk,,1690029077151.fc6cb6b064d3fb792c9e01da5e382760. 2023-07-22T12:31:17,909 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7254): Opening region: {ENCODED => fc6cb6b064d3fb792c9e01da5e382760, NAME => 'backup:system_bulk,,1690029077151.fc6cb6b064d3fb792c9e01da5e382760.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:31:17,909 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:31:17,909 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system_bulk fc6cb6b064d3fb792c9e01da5e382760 2023-07-22T12:31:17,909 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1690029077151.fc6cb6b064d3fb792c9e01da5e382760.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:31:17,909 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7296): checking encryption for fc6cb6b064d3fb792c9e01da5e382760 2023-07-22T12:31:17,910 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7299): checking classloading for fc6cb6b064d3fb792c9e01da5e382760 2023-07-22T12:31:17,911 INFO [StoreOpener-fc6cb6b064d3fb792c9e01da5e382760-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region fc6cb6b064d3fb792c9e01da5e382760 2023-07-22T12:31:17,913 INFO [StoreOpener-fc6cb6b064d3fb792c9e01da5e382760-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc6cb6b064d3fb792c9e01da5e382760 columnFamilyName meta 2023-07-22T12:31:17,913 DEBUG [StoreOpener-fc6cb6b064d3fb792c9e01da5e382760-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:31:17,914 INFO [StoreOpener-fc6cb6b064d3fb792c9e01da5e382760-1 {}] regionserver.HStore(324): Store=fc6cb6b064d3fb792c9e01da5e382760/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:31:17,914 INFO [StoreOpener-fc6cb6b064d3fb792c9e01da5e382760-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region fc6cb6b064d3fb792c9e01da5e382760 2023-07-22T12:31:17,916 INFO [StoreOpener-fc6cb6b064d3fb792c9e01da5e382760-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc6cb6b064d3fb792c9e01da5e382760 columnFamilyName session 2023-07-22T12:31:17,916 DEBUG [StoreOpener-fc6cb6b064d3fb792c9e01da5e382760-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:31:17,916 INFO [StoreOpener-fc6cb6b064d3fb792c9e01da5e382760-1 {}] regionserver.HStore(324): Store=fc6cb6b064d3fb792c9e01da5e382760/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:31:17,917 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system_bulk/fc6cb6b064d3fb792c9e01da5e382760 2023-07-22T12:31:17,918 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system_bulk/fc6cb6b064d3fb792c9e01da5e382760 2023-07-22T12:31:17,919 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system_bulk descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-07-22T12:31:17,921 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1076): writing seq id for fc6cb6b064d3fb792c9e01da5e382760 2023-07-22T12:31:17,923 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system_bulk/fc6cb6b064d3fb792c9e01da5e382760/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:31:17,923 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): Opened fc6cb6b064d3fb792c9e01da5e382760; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10580605120, jitterRate=-0.014604359865188599}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-07-22T12:31:17,924 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(992): Region open journal for fc6cb6b064d3fb792c9e01da5e382760: 2023-07-22T12:31:17,925 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2552): Post open deploy tasks for backup:system_bulk,,1690029077151.fc6cb6b064d3fb792c9e01da5e382760., pid=31, masterSystemTime=1690029077904 2023-07-22T12:31:17,927 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2579): Finished post open deploy task for backup:system_bulk,,1690029077151.fc6cb6b064d3fb792c9e01da5e382760. 2023-07-22T12:31:17,927 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(158): Opened backup:system_bulk,,1690029077151.fc6cb6b064d3fb792c9e01da5e382760. 2023-07-22T12:31:17,927 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=fc6cb6b064d3fb792c9e01da5e382760, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:17,931 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=31, resume processing ppid=30 2023-07-22T12:31:17,931 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=31, ppid=30, state=SUCCESS; OpenRegionProcedure fc6cb6b064d3fb792c9e01da5e382760, server=jenkins-hbase3.apache.org,43639,1690029056327 in 177 msec 2023-07-22T12:31:17,934 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=30, resume processing ppid=29 2023-07-22T12:31:17,934 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=30, ppid=29, state=SUCCESS; TransitRegionStateProcedure table=backup:system_bulk, region=fc6cb6b064d3fb792c9e01da5e382760, ASSIGN in 337 msec 2023-07-22T12:31:17,935 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:31:17,935 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029077935"}]},"ts":"1690029077935"} 2023-07-22T12:31:17,936 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLED in hbase:meta 2023-07-22T12:31:17,939 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:31:17,940 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=29, state=SUCCESS; CreateTableProcedure table=backup:system_bulk in 788 msec 2023-07-22T12:31:18,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-07-22T12:31:18,263 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: backup:system_bulk, procId: 29 completed 2023-07-22T12:31:18,279 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-07-22T12:31:18,351 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:31:18,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029078352 (current time:1690029078352). 2023-07-22T12:31:18,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:31:18,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-07-22T12:31:18,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x551163cc to 127.0.0.1:62657 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:31:18,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f738eb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:31:18,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:31:18,369 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:58160, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:31:18,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x551163cc to 127.0.0.1:62657 2023-07-22T12:31:18,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:31:18,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:31:18,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=32, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:31:18,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-07-22T12:31:18,389 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:31:18,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-07-22T12:31:18,397 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:31:18,400 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:31:18,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-07-22T12:31:18,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-07-22T12:31:18,813 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:31:18,815 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33}] 2023-07-22T12:31:18,823 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33 2023-07-22T12:31:18,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43639 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=33 2023-07-22T12:31:18,988 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. 2023-07-22T12:31:18,991 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2819): Flushing 697cc5ecf0962f5ab9f4126febf7ce33 2/2 column families, dataSize=45 B heapSize=632 B 2023-07-22T12:31:18,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-07-22T12:31:19,065 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/session/956010b3eaa64b4fa82ddea4557fac2c is 49, key is activesession:/session:c/1690029078330/Put/seqid=0 2023-07-22T12:31:19,372 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-07-22T12:31:19,484 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/session/956010b3eaa64b4fa82ddea4557fac2c 2023-07-22T12:31:19,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-07-22T12:31:19,554 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 956010b3eaa64b4fa82ddea4557fac2c 2023-07-22T12:31:19,559 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/session/956010b3eaa64b4fa82ddea4557fac2c as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/956010b3eaa64b4fa82ddea4557fac2c 2023-07-22T12:31:19,568 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/956010b3eaa64b4fa82ddea4557fac2c, entries=1, sequenceid=5, filesize=4.9 K 2023-07-22T12:31:19,572 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(3022): Finished flush of dataSize ~45 B/45, heapSize ~360 B/360, currentSize=0 B/0 for 697cc5ecf0962f5ab9f4126febf7ce33 in 581ms, sequenceid=5, compaction requested=false 2023-07-22T12:31:19,574 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2520): Flush status journal for 697cc5ecf0962f5ab9f4126febf7ce33: 2023-07-22T12:31:19,574 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. for snapshot_backup_system completed. 2023-07-22T12:31:19,575 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33.' region-info for snapshot=snapshot_backup_system 2023-07-22T12:31:19,582 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:31:19,587 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-07-22T12:31:19,587 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/956010b3eaa64b4fa82ddea4557fac2c] hfiles 2023-07-22T12:31:19,587 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/956010b3eaa64b4fa82ddea4557fac2c for snapshot=snapshot_backup_system 2023-07-22T12:31:20,005 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. 2023-07-22T12:31:20,009 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=33 2023-07-22T12:31:20,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster(3994): Remote procedure done, pid=33 2023-07-22T12:31:20,012 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 697cc5ecf0962f5ab9f4126febf7ce33 2023-07-22T12:31:20,014 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33 2023-07-22T12:31:20,018 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=33, resume processing ppid=32 2023-07-22T12:31:20,018 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:31:20,018 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=33, ppid=32, state=SUCCESS; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33 in 1.2000 sec 2023-07-22T12:31:20,020 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:31:20,023 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:31:20,023 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-07-22T12:31:20,027 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_backup_system 2023-07-22T12:31:20,454 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:31:20,468 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:31:20,469 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_backup_system 2023-07-22T12:31:20,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-07-22T12:31:21,304 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:31:21,305 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-07-22T12:31:21,306 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=32, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.9250 sec 2023-07-22T12:31:22,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-07-22T12:31:22,500 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: backup:system, procId: 32 completed 2023-07-22T12:31:22,501 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(115): Backup backup_1690029074871 started at 1690029082501. 2023-07-22T12:31:22,522 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(119): Backup session backup_1690029074871 has been started. 2023-07-22T12:31:22,525 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-07-22T12:31:22,536 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-07-22T12:31:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-07-22T12:31:22,544 INFO [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-07-22T12:31:22,544 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-07-22T12:31:22,544 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-07-22T12:31:22,544 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-07-22T12:31:22,545 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-07-22T12:31:22,545 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:31:22,546 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): regionserver:43639-0x100005e31340001, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-07-22T12:31:22,546 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:22,546 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:31:22,546 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:31:22,547 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:22,547 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-07-22T12:31:22,547 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-07-22T12:31:22,547 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43639-0x100005e31340001, quorum=127.0.0.1:62657, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-07-22T12:31:22,548 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-07-22T12:31:22,548 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:31:22,548 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-07-22T12:31:22,550 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-07-22T12:31:22,550 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-07-22T12:31:22,550 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-07-22T12:31:22,550 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-07-22T12:31:22,551 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-07-22T12:31:22,551 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-07-22T12:31:22,551 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,43639,1690029056327' joining acquired barrier for procedure (rolllog) in zk 2023-07-22T12:31:22,553 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:22,553 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-07-22T12:31:22,553 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:22,553 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:31:22,553 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:31:22,553 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43639-0x100005e31340001, quorum=127.0.0.1:62657, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-07-22T12:31:22,553 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-07-22T12:31:22,553 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:31:22,554 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:31:22,554 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:31:22,554 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:22,555 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:31:22,555 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,43639,1690029056327' joining acquired barrier for procedure 'rolllog' on coordinator 2023-07-22T12:31:22,555 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@2e96af6c[Count = 0] remaining members to acquire global barrier 2023-07-22T12:31:22,555 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-07-22T12:31:22,555 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-07-22T12:31:22,557 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): regionserver:43639-0x100005e31340001, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-07-22T12:31:22,557 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-07-22T12:31:22,557 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-07-22T12:31:22,557 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-07-22T12:31:22,557 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:22,557 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-07-22T12:31:22,558 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-07-22T12:31:22,558 DEBUG [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:22,558 INFO [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1690029058892 highest: 1690029058892 on jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:22,559 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta:.meta(num 1690029059296) roll requested 2023-07-22T12:31:22,570 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029082559.meta, exclude list is [], retry=0 2023-07-22T12:31:22,574 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK] 2023-07-22T12:31:22,577 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029059296.meta with entries=36, filesize=11.13 KB; new WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029082559.meta 2023-07-22T12:31:22,578 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK]] 2023-07-22T12:31:22,579 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029059296.meta is not closed yet, will try archiving it next time 2023-07-22T12:31:22,579 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C43639%2C1690029056327:(num 1690029058892) roll requested 2023-07-22T12:31:22,591 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029082579, exclude list is [], retry=0 2023-07-22T12:31:22,595 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK] 2023-07-22T12:31:22,598 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029058892 with entries=19, filesize=4.02 KB; new WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029082579 2023-07-22T12:31:22,599 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK]] 2023-07-22T12:31:22,599 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029058892 is not closed yet, will try archiving it next time 2023-07-22T12:31:22,619 DEBUG [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 61 2023-07-22T12:31:22,619 INFO [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1690029082579 on jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:22,620 DEBUG [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:31:22,622 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:49802, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-07-22T12:31:22,640 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-07-22T12:31:22,640 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,43639,1690029056327' in zk 2023-07-22T12:31:22,644 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:22,644 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-07-22T12:31:22,644 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:31:22,644 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-07-22T12:31:22,644 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:22,645 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:31:22,645 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:31:22,646 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:31:22,646 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:31:22,646 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:31:22,647 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:22,647 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:31:22,647 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:31:22,648 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:22,648 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,43639,1690029056327': 2023-07-22T12:31:22,648 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,43639,1690029056327' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-07-22T12:31:22,648 INFO [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-07-22T12:31:22,648 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-07-22T12:31:22,648 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-07-22T12:31:22,648 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-07-22T12:31:22,648 INFO [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-07-22T12:31:22,651 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-07-22T12:31:22,651 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): regionserver:43639-0x100005e31340001, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-07-22T12:31:22,651 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): regionserver:43639-0x100005e31340001, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-07-22T12:31:22,651 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-07-22T12:31:22,651 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-07-22T12:31:22,651 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:31:22,651 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:31:22,651 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:31:22,651 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:31:22,651 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:31:22,651 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:31:22,652 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:22,652 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:31:22,652 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:31:22,655 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:31:22,656 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:31:22,656 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:22,656 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:22,657 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:31:22,657 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:31:22,657 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:22,666 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:22,666 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): regionserver:43639-0x100005e31340001, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-07-22T12:31:22,667 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): regionserver:43639-0x100005e31340001, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-07-22T12:31:22,667 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-07-22T12:31:22,667 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:31:22,667 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-07-22T12:31:22,667 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:31:22,667 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:22,667 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-07-22T12:31:22,667 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:31:22,667 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-07-22T12:31:22,667 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-07-22T12:31:22,667 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-07-22T12:31:22,668 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-07-22T12:31:22,668 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:31:22,668 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:31:22,670 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] client.HBaseAdmin(2751): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-07-22T12:31:22,670 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] client.HBaseAdmin(2759): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-07-22T12:31:22,770 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] client.HBaseAdmin(2765): Getting current status of procedure from master... 2023-07-22T12:31:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-07-22T12:31:22,781 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1690029082780_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } 2023-07-22T12:31:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029082781 (current time:1690029082781). 2023-07-22T12:31:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:31:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1690029082780_ns2_test-16900290656251 VERSION not specified, setting to 2 2023-07-22T12:31:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x69985e19 to 127.0.0.1:62657 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:31:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fef4798, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:31:22,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:31:22,791 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:55834, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:31:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x69985e19 to 127.0.0.1:62657 2023-07-22T12:31:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:31:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:31:22,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=34, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1690029082780_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } 2023-07-22T12:31:22,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1690029082780_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 }, snapshot procedure id = 34 2023-07-22T12:31:22,796 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1690029082780_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:31:22,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-07-22T12:31:22,797 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1690029082780_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:31:22,799 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1690029082780_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:31:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-07-22T12:31:22,958 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system_bulk' 2023-07-22T12:31:23,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-07-22T12:31:23,210 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1690029082780_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:31:23,210 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure 0112d97ae6696b92341dfd7ff8824a0a}] 2023-07-22T12:31:23,213 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure 0112d97ae6696b92341dfd7ff8824a0a 2023-07-22T12:31:23,225 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-16900290656253 2023-07-22T12:31:23,225 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns4_table_test-16900290656253 Metrics about Tables on a single HBase RegionServer 2023-07-22T12:31:23,226 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-07-22T12:31:23,226 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2023-07-22T12:31:23,226 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2023-07-22T12:31:23,226 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk Metrics about Tables on a single HBase RegionServer 2023-07-22T12:31:23,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43639 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=35 2023-07-22T12:31:23,365 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-16900290656251,,1690029068374.0112d97ae6696b92341dfd7ff8824a0a. 2023-07-22T12:31:23,366 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2819): Flushing 0112d97ae6696b92341dfd7ff8824a0a 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-07-22T12:31:23,385 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/0112d97ae6696b92341dfd7ff8824a0a/.tmp/f/a0d603de2bbc4f1988dbc8924efa3383 is 37, key is row10/f:q1/1690029070502/Put/seqid=0 2023-07-22T12:31:23,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-07-22T12:31:23,797 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/0112d97ae6696b92341dfd7ff8824a0a/.tmp/f/a0d603de2bbc4f1988dbc8924efa3383 2023-07-22T12:31:23,807 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a0d603de2bbc4f1988dbc8924efa3383 2023-07-22T12:31:23,808 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/0112d97ae6696b92341dfd7ff8824a0a/.tmp/f/a0d603de2bbc4f1988dbc8924efa3383 as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/0112d97ae6696b92341dfd7ff8824a0a/f/a0d603de2bbc4f1988dbc8924efa3383 2023-07-22T12:31:23,815 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/0112d97ae6696b92341dfd7ff8824a0a/f/a0d603de2bbc4f1988dbc8924efa3383, entries=99, sequenceid=103, filesize=8.2 K 2023-07-22T12:31:23,817 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 0112d97ae6696b92341dfd7ff8824a0a in 451ms, sequenceid=103, compaction requested=false 2023-07-22T12:31:23,818 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2520): Flush status journal for 0112d97ae6696b92341dfd7ff8824a0a: 2023-07-22T12:31:23,818 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-16900290656251,,1690029068374.0112d97ae6696b92341dfd7ff8824a0a. for snapshot_1690029082780_ns2_test-16900290656251 completed. 2023-07-22T12:31:23,818 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(240): Storing 'ns2:test-16900290656251,,1690029068374.0112d97ae6696b92341dfd7ff8824a0a.' region-info for snapshot=snapshot_1690029082780_ns2_test-16900290656251 2023-07-22T12:31:23,818 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:31:23,818 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/0112d97ae6696b92341dfd7ff8824a0a/f/a0d603de2bbc4f1988dbc8924efa3383] hfiles 2023-07-22T12:31:23,818 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/0112d97ae6696b92341dfd7ff8824a0a/f/a0d603de2bbc4f1988dbc8924efa3383 for snapshot=snapshot_1690029082780_ns2_test-16900290656251 2023-07-22T12:31:23,828 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-16900290656251,,1690029068374.0112d97ae6696b92341dfd7ff8824a0a. 2023-07-22T12:31:23,828 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=35 2023-07-22T12:31:23,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster(3994): Remote procedure done, pid=35 2023-07-22T12:31:23,829 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1690029082780_ns2_test-16900290656251 on region 0112d97ae6696b92341dfd7ff8824a0a 2023-07-22T12:31:23,829 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure 0112d97ae6696b92341dfd7ff8824a0a 2023-07-22T12:31:23,832 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=35, resume processing ppid=34 2023-07-22T12:31:23,832 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=35, ppid=34, state=SUCCESS; SnapshotRegionProcedure 0112d97ae6696b92341dfd7ff8824a0a in 620 msec 2023-07-22T12:31:23,832 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1690029082780_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:31:23,833 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1690029082780_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:31:23,834 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1690029082780_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:31:23,834 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1690029082780_ns2_test-16900290656251 2023-07-22T12:31:23,834 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_1690029082780_ns2_test-16900290656251 2023-07-22T12:31:23,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-07-22T12:31:24,249 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1690029082780_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:31:24,257 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1690029082780_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:31:24,258 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_1690029082780_ns2_test-16900290656251 to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029082780_ns2_test-16900290656251 2023-07-22T12:31:24,690 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1690029082780_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:31:24,690 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1690029082780_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 }, snapshot procedure id = 34 2023-07-22T12:31:24,692 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=34, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1690029082780_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } in 1.8980 sec 2023-07-22T12:31:24,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-07-22T12:31:24,904 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: ns2:test-16900290656251, procId: 34 completed 2023-07-22T12:31:24,904 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1690029084904_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } 2023-07-22T12:31:24,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029084904 (current time:1690029084904). 2023-07-22T12:31:24,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:31:24,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1690029084904_ns3_test-16900290656252 VERSION not specified, setting to 2 2023-07-22T12:31:24,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x17a9f482 to 127.0.0.1:62657 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:31:24,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cea18f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:31:24,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:31:24,913 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:55846, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:31:24,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x17a9f482 to 127.0.0.1:62657 2023-07-22T12:31:24,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:31:24,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:31:24,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=36, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1690029084904_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } 2023-07-22T12:31:24,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1690029084904_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 }, snapshot procedure id = 36 2023-07-22T12:31:24,918 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1690029084904_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:31:24,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=36 2023-07-22T12:31:24,919 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1690029084904_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:31:24,921 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1690029084904_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:31:25,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=36 2023-07-22T12:31:25,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=36 2023-07-22T12:31:25,334 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1690029084904_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:31:25,334 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE; SnapshotRegionProcedure 335c94783192a14614120d955d2aa716}] 2023-07-22T12:31:25,336 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=37, ppid=36, state=RUNNABLE; SnapshotRegionProcedure 335c94783192a14614120d955d2aa716 2023-07-22T12:31:25,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43639 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=37 2023-07-22T12:31:25,489 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-16900290656252,,1690029070575.335c94783192a14614120d955d2aa716. 2023-07-22T12:31:25,490 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2520): Flush status journal for 335c94783192a14614120d955d2aa716: 2023-07-22T12:31:25,490 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-16900290656252,,1690029070575.335c94783192a14614120d955d2aa716. for snapshot_1690029084904_ns3_test-16900290656252 completed. 2023-07-22T12:31:25,490 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(240): Storing 'ns3:test-16900290656252,,1690029070575.335c94783192a14614120d955d2aa716.' region-info for snapshot=snapshot_1690029084904_ns3_test-16900290656252 2023-07-22T12:31:25,490 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:31:25,490 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-07-22T12:31:25,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=36 2023-07-22T12:31:25,901 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-16900290656252,,1690029070575.335c94783192a14614120d955d2aa716. 2023-07-22T12:31:25,901 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=37 2023-07-22T12:31:25,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster(3994): Remote procedure done, pid=37 2023-07-22T12:31:25,901 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1690029084904_ns3_test-16900290656252 on region 335c94783192a14614120d955d2aa716 2023-07-22T12:31:25,902 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=37, ppid=36, state=RUNNABLE; SnapshotRegionProcedure 335c94783192a14614120d955d2aa716 2023-07-22T12:31:25,905 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=37, resume processing ppid=36 2023-07-22T12:31:25,905 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=37, ppid=36, state=SUCCESS; SnapshotRegionProcedure 335c94783192a14614120d955d2aa716 in 569 msec 2023-07-22T12:31:25,905 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1690029084904_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:31:25,906 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1690029084904_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:31:25,907 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1690029084904_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:31:25,907 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1690029084904_ns3_test-16900290656252 2023-07-22T12:31:25,908 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_1690029084904_ns3_test-16900290656252 2023-07-22T12:31:26,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=36 2023-07-22T12:31:26,326 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1690029084904_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:31:26,332 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1690029084904_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:31:26,333 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_1690029084904_ns3_test-16900290656252 to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029084904_ns3_test-16900290656252 2023-07-22T12:31:27,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=36 2023-07-22T12:31:27,158 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1690029084904_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:31:27,158 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1690029084904_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 }, snapshot procedure id = 36 2023-07-22T12:31:27,160 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=36, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1690029084904_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } in 2.2430 sec 2023-07-22T12:31:29,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=36 2023-07-22T12:31:29,027 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: ns3:test-16900290656252, procId: 36 completed 2023-07-22T12:31:29,027 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1690029074871 2023-07-22T12:31:29,027 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-07-22T12:31:29,029 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2023-07-22T12:31:29,031 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1690029074871_ns2:test-16900290656251 2023-07-22T12:31:29,031 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1690029082780_ns2_test-16900290656251 to hdfs://localhost:34745/backupUT/backup_1690029074871/ns2/test-16900290656251/ 2023-07-22T12:31:29,041 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-07-22T12:31:29,071 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:34745, inputRoot=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947 2023-07-22T12:31:29,071 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,37923,1690029053050_-1389991531_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:34745/backupUT/backup_1690029074871/ns2/test-16900290656251, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:34745/backupUT/backup_1690029074871/ns2/test-16900290656251/.hbase-snapshot/.tmp/snapshot_1690029082780_ns2_test-16900290656251 2023-07-22T12:31:29,071 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:34745, inputRoot=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947, snapshotDir=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029082780_ns2_test-16900290656251. 2023-07-22T12:31:29,083 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029082780_ns2_test-16900290656251 to hdfs://localhost:34745/backupUT/backup_1690029074871/ns2/test-16900290656251/.hbase-snapshot/.tmp/snapshot_1690029082780_ns2_test-16900290656251 2023-07-22T12:31:29,508 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:29,508 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:29,509 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:29,509 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:30,549 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/hadoop-4486556757552802189.jar 2023-07-22T12:31:30,549 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:30,550 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:30,550 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:30,551 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:30,551 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:30,552 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:30,552 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:30,552 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:30,553 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:30,554 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:31:30,554 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-07-22T12:31:30,555 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-07-22T12:31:30,555 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-07-22T12:31:30,556 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-07-22T12:31:30,556 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-07-22T12:31:30,557 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-07-22T12:31:30,557 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-07-22T12:31:30,558 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-07-22T12:31:30,558 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-07-22T12:31:30,558 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-07-22T12:31:30,559 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-07-22T12:31:30,563 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:31:30,563 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:31:30,564 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:31:30,564 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:31:30,564 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:31:30,565 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:31:30,565 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:31:30,626 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-07-22T12:31:30,648 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1690029082780_ns2_test-16900290656251' hfile list 2023-07-22T12:31:30,659 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-07-22T12:31:30,896 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029090839/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-07-22T12:31:30,896 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029090839/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-07-22T12:31:31,057 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-07-22T12:31:31,115 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/0112d97ae6696b92341dfd7ff8824a0a/f/a0d603de2bbc4f1988dbc8924efa3383 output=hdfs://localhost:34745/backupUT/backup_1690029074871/ns2/test-16900290656251/archive/data/ns2/test-16900290656251/0112d97ae6696b92341dfd7ff8824a0a/f/a0d603de2bbc4f1988dbc8924efa3383 2023-07-22T12:31:31,116 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 8.007M/sec 2023-07-22T12:31:31,962 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-07-22T12:31:31,963 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-07-22T12:31:31,972 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1690029082780_ns2_test-16900290656251 2023-07-22T12:31:31,973 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1690029082780_ns2_test-16900290656251 finished. 2023-07-22T12:31:31,973 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1690029074871_ns3:test-16900290656252 2023-07-22T12:31:31,973 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1690029084904_ns3_test-16900290656252 to hdfs://localhost:34745/backupUT/backup_1690029074871/ns3/test-16900290656252/ 2023-07-22T12:31:31,973 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-07-22T12:31:31,997 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:34745, inputRoot=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947 2023-07-22T12:31:31,997 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,37923,1690029053050_-326153336_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:34745/backupUT/backup_1690029074871/ns3/test-16900290656252, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:34745/backupUT/backup_1690029074871/ns3/test-16900290656252/.hbase-snapshot/.tmp/snapshot_1690029084904_ns3_test-16900290656252 2023-07-22T12:31:31,997 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:34745, inputRoot=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947, snapshotDir=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029084904_ns3_test-16900290656252. 2023-07-22T12:31:32,004 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029084904_ns3_test-16900290656252 to hdfs://localhost:34745/backupUT/backup_1690029074871/ns3/test-16900290656252/.hbase-snapshot/.tmp/snapshot_1690029084904_ns3_test-16900290656252 2023-07-22T12:31:32,429 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:32,430 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:32,430 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:32,431 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:33,386 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-07-22T12:31:33,388 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:42004, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-07-22T12:31:33,477 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/hadoop-9150435741167468948.jar 2023-07-22T12:31:33,478 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:33,478 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:33,479 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:33,479 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:33,480 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:33,481 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:33,481 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:33,481 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:33,482 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:33,483 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:31:33,483 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-07-22T12:31:33,484 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-07-22T12:31:33,484 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-07-22T12:31:33,484 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-07-22T12:31:33,485 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-07-22T12:31:33,485 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-07-22T12:31:33,486 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-07-22T12:31:33,486 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-07-22T12:31:33,487 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-07-22T12:31:33,487 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-07-22T12:31:33,488 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-07-22T12:31:33,488 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:31:33,489 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:31:33,490 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:31:33,490 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:31:33,490 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:31:33,491 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:31:33,492 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:31:33,521 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-07-22T12:31:33,538 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1690029084904_ns3_test-16900290656252' hfile list 2023-07-22T12:31:33,629 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029093590/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-07-22T12:31:33,629 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029093590/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-07-22T12:31:34,661 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-07-22T12:31:34,663 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-07-22T12:31:34,669 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1690029084904_ns3_test-16900290656252 2023-07-22T12:31:34,669 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1690029084904_ns3_test-16900290656252 finished. 2023-07-22T12:31:34,670 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(1014): ns2:test-16900290656251 2023-07-22T12:31:34,670 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(1014): ns3:test-16900290656252 2023-07-22T12:31:34,744 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029074871 2023-07-22T12:31:34,744 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-07-22T12:31:35,157 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:34745/backupUT/backup_1690029074871/.backup.manifest 2023-07-22T12:31:35,157 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029074871 2023-07-22T12:31:35,157 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-07-22T12:31:35,564 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:34745/backupUT/backup_1690029074871/.backup.manifest 2023-07-22T12:31:35,565 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(358): Backup backup_1690029074871 finished: type=FULL,tablelist=ns2:test-16900290656251;ns3:test-16900290656252,targetRootDir=hdfs://localhost:34745/backupUT,startts=1690029082501,completets=1690029094741,bytescopied=0 2023-07-22T12:31:35,565 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-07-22T12:31:35,565 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1690029082780_ns2_test-16900290656251 2023-07-22T12:31:35,575 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1690029082780_ns2_test-16900290656251" 2023-07-22T12:31:35,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1690029082780_ns2_test-16900290656251 2023-07-22T12:31:35,581 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1690029082780_ns2_test-16900290656251 for backup backup_1690029074871 succeeded. 2023-07-22T12:31:35,581 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1690029084904_ns3_test-16900290656252 2023-07-22T12:31:35,582 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1690029084904_ns3_test-16900290656252" 2023-07-22T12:31:35,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1690029084904_ns3_test-16900290656252 2023-07-22T12:31:35,587 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1690029084904_ns3_test-16900290656252 for backup backup_1690029074871 succeeded. 2023-07-22T12:31:35,588 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-07-22T12:31:35,601 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-07-22T12:31:35,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-07-22T12:31:35,605 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-07-22T12:31:35,612 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-07-22T12:31:35,629 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(376): Backup backup_1690029074871 completed. 2023-07-22T12:31:35,629 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] client.ConnectionImplementation(2095): Closing master protocol: MasterService 2023-07-22T12:31:35,630 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x63fd3bc4 to 127.0.0.1:62657 2023-07-22T12:31:35,630 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:31:35,657 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34745/backupUT/backup_1690029074871/.backup.manifest 2023-07-22T12:31:35,660 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029074871/.backup.manifest 2023-07-22T12:31:35,660 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34745/backupUT/backup_1690029074871/.backup.manifest 2023-07-22T12:31:35,662 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029074871/.backup.manifest 2023-07-22T12:31:35,668 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34745/backupUT/backup_1690029074871/.backup.manifest 2023-07-22T12:31:35,671 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029074871/.backup.manifest 2023-07-22T12:31:35,671 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(148): Restoring 'ns2:test-16900290656251' to 'ns2:test-16900290656251' from full backup image hdfs://localhost:34745/backupUT/backup_1690029074871/ns2/test-16900290656251 2023-07-22T12:31:35,683 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] util.RestoreTool(476): Truncating exising target table 'ns2:test-16900290656251', preserving region splits 2023-07-22T12:31:35,684 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] client.HBaseAdmin$18(963): Started disable of ns2:test-16900290656251 2023-07-22T12:31:35,689 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster$12(2663): Client=jenkins//172.31.12.81 disable ns2:test-16900290656251 2023-07-22T12:31:35,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=38, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=ns2:test-16900290656251 2023-07-22T12:31:35,702 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16900290656251","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029095701"}]},"ts":"1690029095701"} 2023-07-22T12:31:35,704 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16900290656251, state=DISABLING in hbase:meta 2023-07-22T12:31:35,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=38 2023-07-22T12:31:35,705 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(293): Set ns2:test-16900290656251 to state=DISABLING 2023-07-22T12:31:35,707 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns2:test-16900290656251, region=0112d97ae6696b92341dfd7ff8824a0a, UNASSIGN}] 2023-07-22T12:31:35,709 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns2:test-16900290656251, region=0112d97ae6696b92341dfd7ff8824a0a, UNASSIGN 2023-07-22T12:31:35,710 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=39 updating hbase:meta row=0112d97ae6696b92341dfd7ff8824a0a, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:35,712 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE; CloseRegionProcedure 0112d97ae6696b92341dfd7ff8824a0a, server=jenkins-hbase3.apache.org,43639,1690029056327}] 2023-07-22T12:31:35,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=38 2023-07-22T12:31:35,872 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(119): Close 0112d97ae6696b92341dfd7ff8824a0a 2023-07-22T12:31:35,872 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1668): Closing 0112d97ae6696b92341dfd7ff8824a0a, disabling compactions & flushes 2023-07-22T12:31:35,872 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1690): Closing region ns2:test-16900290656251,,1690029068374.0112d97ae6696b92341dfd7ff8824a0a. 2023-07-22T12:31:35,872 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16900290656251,,1690029068374.0112d97ae6696b92341dfd7ff8824a0a. 2023-07-22T12:31:35,872 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16900290656251,,1690029068374.0112d97ae6696b92341dfd7ff8824a0a. after waiting 0 ms 2023-07-22T12:31:35,872 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16900290656251,,1690029068374.0112d97ae6696b92341dfd7ff8824a0a. 2023-07-22T12:31:35,875 DEBUG [StoreFileCloser-0112d97ae6696b92341dfd7ff8824a0a-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a0d603de2bbc4f1988dbc8924efa3383 2023-07-22T12:31:35,880 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/0112d97ae6696b92341dfd7ff8824a0a/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2023-07-22T12:31:35,881 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:31:35,882 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1909): Closed ns2:test-16900290656251,,1690029068374.0112d97ae6696b92341dfd7ff8824a0a. 2023-07-22T12:31:35,882 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1622): Region close journal for 0112d97ae6696b92341dfd7ff8824a0a: 2023-07-22T12:31:35,884 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(163): Closed 0112d97ae6696b92341dfd7ff8824a0a 2023-07-22T12:31:35,885 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=39 updating hbase:meta row=0112d97ae6696b92341dfd7ff8824a0a, regionState=CLOSED 2023-07-22T12:31:35,889 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=40, resume processing ppid=39 2023-07-22T12:31:35,889 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=40, ppid=39, state=SUCCESS; CloseRegionProcedure 0112d97ae6696b92341dfd7ff8824a0a, server=jenkins-hbase3.apache.org,43639,1690029056327 in 174 msec 2023-07-22T12:31:35,891 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=39, resume processing ppid=38 2023-07-22T12:31:35,891 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=39, ppid=38, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-16900290656251, region=0112d97ae6696b92341dfd7ff8824a0a, UNASSIGN in 182 msec 2023-07-22T12:31:35,892 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16900290656251","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029095892"}]},"ts":"1690029095892"} 2023-07-22T12:31:35,894 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16900290656251, state=DISABLED in hbase:meta 2023-07-22T12:31:35,897 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(305): Set ns2:test-16900290656251 to state=DISABLED 2023-07-22T12:31:35,899 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=38, state=SUCCESS; DisableTableProcedure table=ns2:test-16900290656251 in 207 msec 2023-07-22T12:31:36,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=38 2023-07-22T12:31:36,007 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] client.HBaseAdmin$TableFuture(3590): Operation: DISABLE, Table Name: ns2:test-16900290656251, procId: 38 completed 2023-07-22T12:31:36,009 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] client.HBaseAdmin$16(843): Started truncating ns2:test-16900290656251 2023-07-22T12:31:36,016 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster$6(2479): Client=jenkins//172.31.12.81 truncate ns2:test-16900290656251 2023-07-22T12:31:36,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=41, state=RUNNABLE:TRUNCATE_TABLE_PRE_OPERATION; TruncateTableProcedure (table=ns2:test-16900290656251 preserveSplits=true) 2023-07-22T12:31:36,028 DEBUG [PEWorker-2 {}] procedure.TruncateTableProcedure(87): waiting for 'ns2:test-16900290656251' regions in transition 2023-07-22T12:31:36,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=41 2023-07-22T12:31:36,036 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/0112d97ae6696b92341dfd7ff8824a0a 2023-07-22T12:31:36,041 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/0112d97ae6696b92341dfd7ff8824a0a/f, FileablePath, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/0112d97ae6696b92341dfd7ff8824a0a/recovered.edits] 2023-07-22T12:31:36,048 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/0112d97ae6696b92341dfd7ff8824a0a/f/a0d603de2bbc4f1988dbc8924efa3383 to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/archive/data/ns2/test-16900290656251/0112d97ae6696b92341dfd7ff8824a0a/f/a0d603de2bbc4f1988dbc8924efa3383 2023-07-22T12:31:36,053 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/0112d97ae6696b92341dfd7ff8824a0a/recovered.edits/106.seqid to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/archive/data/ns2/test-16900290656251/0112d97ae6696b92341dfd7ff8824a0a/recovered.edits/106.seqid 2023-07-22T12:31:36,056 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/0112d97ae6696b92341dfd7ff8824a0a 2023-07-22T12:31:36,056 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(293): Archived ns2:test-16900290656251 regions 2023-07-22T12:31:36,070 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of ns2:test-16900290656251 from hbase:meta 2023-07-22T12:31:36,073 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(387): Removing 'ns2:test-16900290656251' descriptor. 2023-07-22T12:31:36,073 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(377): Removing 'ns2:test-16900290656251' from region states. 2023-07-22T12:31:36,074 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns2:test-16900290656251,,1690029068374.0112d97ae6696b92341dfd7ff8824a0a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1690029096073"}]},"ts":"9223372036854775807"} 2023-07-22T12:31:36,076 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-07-22T12:31:36,076 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 0112d97ae6696b92341dfd7ff8824a0a, NAME => 'ns2:test-16900290656251,,1690029068374.0112d97ae6696b92341dfd7ff8824a0a.', STARTKEY => '', ENDKEY => ''}] 2023-07-22T12:31:36,076 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(381): Marking 'ns2:test-16900290656251' as deleted. 2023-07-22T12:31:36,077 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns2:test-16900290656251","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1690029096076"}]},"ts":"9223372036854775807"} 2023-07-22T12:31:36,079 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1678): Deleted table ns2:test-16900290656251 state from META 2023-07-22T12:31:36,092 INFO [RegionOpenAndInit-ns2:test-16900290656251-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => b8196e3466217e5359c3acacaaeb9e61, NAME => 'ns2:test-16900290656251,,1690029096058.b8196e3466217e5359c3acacaaeb9e61.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-16900290656251', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947 2023-07-22T12:31:36,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=41 2023-07-22T12:31:36,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=41 2023-07-22T12:31:36,501 DEBUG [RegionOpenAndInit-ns2:test-16900290656251-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:test-16900290656251,,1690029096058.b8196e3466217e5359c3acacaaeb9e61.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:31:36,501 DEBUG [RegionOpenAndInit-ns2:test-16900290656251-pool-0 {}] regionserver.HRegion(1668): Closing b8196e3466217e5359c3acacaaeb9e61, disabling compactions & flushes 2023-07-22T12:31:36,501 INFO [RegionOpenAndInit-ns2:test-16900290656251-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:test-16900290656251,,1690029096058.b8196e3466217e5359c3acacaaeb9e61. 2023-07-22T12:31:36,501 DEBUG [RegionOpenAndInit-ns2:test-16900290656251-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16900290656251,,1690029096058.b8196e3466217e5359c3acacaaeb9e61. 2023-07-22T12:31:36,501 DEBUG [RegionOpenAndInit-ns2:test-16900290656251-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16900290656251,,1690029096058.b8196e3466217e5359c3acacaaeb9e61. after waiting 0 ms 2023-07-22T12:31:36,501 DEBUG [RegionOpenAndInit-ns2:test-16900290656251-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16900290656251,,1690029096058.b8196e3466217e5359c3acacaaeb9e61. 2023-07-22T12:31:36,501 INFO [RegionOpenAndInit-ns2:test-16900290656251-pool-0 {}] regionserver.HRegion(1909): Closed ns2:test-16900290656251,,1690029096058.b8196e3466217e5359c3acacaaeb9e61. 2023-07-22T12:31:36,501 DEBUG [RegionOpenAndInit-ns2:test-16900290656251-pool-0 {}] regionserver.HRegion(1622): Region close journal for b8196e3466217e5359c3acacaaeb9e61: 2023-07-22T12:31:36,502 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:test-16900290656251,,1690029096058.b8196e3466217e5359c3acacaaeb9e61.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1690029096502"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690029096502"}]},"ts":"1690029096502"} 2023-07-22T12:31:36,504 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:31:36,505 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16900290656251","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029096505"}]},"ts":"1690029096505"} 2023-07-22T12:31:36,506 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16900290656251, state=ENABLING in hbase:meta 2023-07-22T12:31:36,511 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=42, ppid=41, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16900290656251, region=b8196e3466217e5359c3acacaaeb9e61, ASSIGN}] 2023-07-22T12:31:36,513 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=42, ppid=41, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16900290656251, region=b8196e3466217e5359c3acacaaeb9e61, ASSIGN 2023-07-22T12:31:36,514 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=42, ppid=41, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:test-16900290656251, region=b8196e3466217e5359c3acacaaeb9e61, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,43639,1690029056327; forceNewPlan=false, retain=false 2023-07-22T12:31:36,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=41 2023-07-22T12:31:36,665 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=42 updating hbase:meta row=b8196e3466217e5359c3acacaaeb9e61, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:36,668 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE; OpenRegionProcedure b8196e3466217e5359c3acacaaeb9e61, server=jenkins-hbase3.apache.org,43639,1690029056327}] 2023-07-22T12:31:36,824 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(130): Open ns2:test-16900290656251,,1690029096058.b8196e3466217e5359c3acacaaeb9e61. 2023-07-22T12:31:36,825 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7254): Opening region: {ENCODED => b8196e3466217e5359c3acacaaeb9e61, NAME => 'ns2:test-16900290656251,,1690029096058.b8196e3466217e5359c3acacaaeb9e61.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:31:36,825 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:31:36,825 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16900290656251 b8196e3466217e5359c3acacaaeb9e61 2023-07-22T12:31:36,825 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(888): Instantiated ns2:test-16900290656251,,1690029096058.b8196e3466217e5359c3acacaaeb9e61.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:31:36,825 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7296): checking encryption for b8196e3466217e5359c3acacaaeb9e61 2023-07-22T12:31:36,825 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7299): checking classloading for b8196e3466217e5359c3acacaaeb9e61 2023-07-22T12:31:36,827 INFO [StoreOpener-b8196e3466217e5359c3acacaaeb9e61-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region b8196e3466217e5359c3acacaaeb9e61 2023-07-22T12:31:36,828 INFO [StoreOpener-b8196e3466217e5359c3acacaaeb9e61-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b8196e3466217e5359c3acacaaeb9e61 columnFamilyName f 2023-07-22T12:31:36,829 DEBUG [StoreOpener-b8196e3466217e5359c3acacaaeb9e61-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:31:36,829 INFO [StoreOpener-b8196e3466217e5359c3acacaaeb9e61-1 {}] regionserver.HStore(324): Store=b8196e3466217e5359c3acacaaeb9e61/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:31:36,830 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/b8196e3466217e5359c3acacaaeb9e61 2023-07-22T12:31:36,831 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/b8196e3466217e5359c3acacaaeb9e61 2023-07-22T12:31:36,834 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1076): writing seq id for b8196e3466217e5359c3acacaaeb9e61 2023-07-22T12:31:36,836 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/b8196e3466217e5359c3acacaaeb9e61/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:31:36,837 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): Opened b8196e3466217e5359c3acacaaeb9e61; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11722314720, jitterRate=0.09172563254833221}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-07-22T12:31:36,837 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(992): Region open journal for b8196e3466217e5359c3acacaaeb9e61: 2023-07-22T12:31:36,838 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2552): Post open deploy tasks for ns2:test-16900290656251,,1690029096058.b8196e3466217e5359c3acacaaeb9e61., pid=43, masterSystemTime=1690029096820 2023-07-22T12:31:36,840 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2579): Finished post open deploy task for ns2:test-16900290656251,,1690029096058.b8196e3466217e5359c3acacaaeb9e61. 2023-07-22T12:31:36,840 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(158): Opened ns2:test-16900290656251,,1690029096058.b8196e3466217e5359c3acacaaeb9e61. 2023-07-22T12:31:36,841 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=42 updating hbase:meta row=b8196e3466217e5359c3acacaaeb9e61, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:36,845 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=43, resume processing ppid=42 2023-07-22T12:31:36,845 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=43, ppid=42, state=SUCCESS; OpenRegionProcedure b8196e3466217e5359c3acacaaeb9e61, server=jenkins-hbase3.apache.org,43639,1690029056327 in 175 msec 2023-07-22T12:31:36,847 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=42, resume processing ppid=41 2023-07-22T12:31:36,847 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=42, ppid=41, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-16900290656251, region=b8196e3466217e5359c3acacaaeb9e61, ASSIGN in 334 msec 2023-07-22T12:31:36,848 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16900290656251","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029096847"}]},"ts":"1690029096847"} 2023-07-22T12:31:36,849 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16900290656251, state=ENABLED in hbase:meta 2023-07-22T12:31:36,851 DEBUG [PEWorker-3 {}] procedure.TruncateTableProcedure(145): truncate 'ns2:test-16900290656251' completed 2023-07-22T12:31:36,853 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=41, state=SUCCESS; TruncateTableProcedure (table=ns2:test-16900290656251 preserveSplits=true) in 832 msec 2023-07-22T12:31:37,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=41 2023-07-22T12:31:37,137 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] client.HBaseAdmin$TableFuture(3590): Operation: TRUNCATE, Table Name: ns2:test-16900290656251, procId: 41 completed 2023-07-22T12:31:37,144 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:34745/backupUT/backup_1690029074871/ns2/test-16900290656251/archive/data/ns2/test-16900290656251/0112d97ae6696b92341dfd7ff8824a0a from hbase tables ns2:test-16900290656251 to tables ns2:test-16900290656251 2023-07-22T12:31:37,144 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceRestoreJob(72): Restore ns2:test-16900290656251 into ns2:test-16900290656251 2023-07-22T12:31:37,147 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-ns2-test-16900290656251-1690029097144 from hdfs://localhost:34745/backupUT/backup_1690029074871/ns2/test-16900290656251/archive/data/ns2/test-16900290656251/0112d97ae6696b92341dfd7ff8824a0a 2023-07-22T12:31:37,150 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x451e4004 to 127.0.0.1:62657 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:31:37,155 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@264566b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:31:37,157 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:31:37,158 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:42008, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:31:37,172 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-07-22T12:31:37,172 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:test-16900290656251 2023-07-22T12:31:37,177 DEBUG [hconnection-0x3d398b52-shared-pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:31:37,178 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:49180, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:31:37,181 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:test-16900290656251: 2023-07-22T12:31:37,182 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-07-22T12:31:37,182 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/partitions_00642e6e-f443-40f4-b35a-8bc37e5f1888 2023-07-22T12:31:37,624 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:37,624 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:37,625 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:37,625 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:38,609 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/hadoop-6106168650837958339.jar 2023-07-22T12:31:38,610 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:38,610 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:38,610 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:38,611 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:38,611 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:38,612 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:38,612 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:38,613 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:38,613 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:38,614 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:31:38,614 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-07-22T12:31:38,615 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-07-22T12:31:38,615 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-07-22T12:31:38,616 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-07-22T12:31:38,616 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-07-22T12:31:38,617 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-07-22T12:31:38,617 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-07-22T12:31:38,617 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-07-22T12:31:38,618 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-07-22T12:31:38,618 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-07-22T12:31:38,619 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-07-22T12:31:38,622 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:38,623 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:38,623 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:38,624 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:38,624 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:38,624 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:38,625 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:31:38,625 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: ns2:test-16900290656251 2023-07-22T12:31:38,625 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] client.ConnectionImplementation(2095): Closing master protocol: MasterService 2023-07-22T12:31:38,625 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x451e4004 to 127.0.0.1:62657 2023-07-22T12:31:38,626 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:31:38,626 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-07-22T12:31:38,626 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-07-22T12:31:38,626 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:31:38,632 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-07-22T12:31:38,638 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-07-22T12:31:38,728 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029098692/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-07-22T12:31:38,728 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029098692/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-07-22T12:31:38,821 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:34745/backupUT/backup_1690029074871/ns2/test-16900290656251/archive/data/ns2/test-16900290656251/0112d97ae6696b92341dfd7ff8824a0a/f/a0d603de2bbc4f1988dbc8924efa3383 2023-07-22T12:31:38,827 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-07-22T12:31:38,831 DEBUG [LocalJobRunner Map Task Executor #0 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a0d603de2bbc4f1988dbc8924efa3383 2023-07-22T12:31:38,954 DEBUG [pool-200-thread-1 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x228f5edf to 127.0.0.1:62657 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:31:38,963 DEBUG [pool-200-thread-1 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b5c5d58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:31:38,965 DEBUG [hconnection-0x2c17faa1-metaLookup-shared--pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:31:38,967 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:49196, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:31:38,973 DEBUG [pool-200-thread-1 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x228f5edf to 127.0.0.1:62657 2023-07-22T12:31:38,973 DEBUG [pool-200-thread-1 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:31:38,973 DEBUG [pool-200-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-07-22T12:31:38,973 DEBUG [pool-200-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-07-22T12:31:38,991 DEBUG [pool-200-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-ns2-test-16900290656251-1690029097144/_temporary/0/_temporary/attempt_local1007411544_0003_r_000000_0/f/d70ff17ae32b43b7898a7a0fa975c94f is 37, key is row10/f:q1/1690029070502/Put/seqid=0 2023-07-22T12:31:38,996 WARN [Thread-821 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:43639] Specified favored nodes: [jenkins-hbase3.apache.org:43639] 2023-07-22T12:31:39,792 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-ns2-test-16900290656251-1690029097144 2023-07-22T12:31:39,792 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x4473ba0b to 127.0.0.1:62657 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:31:39,798 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ab987a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:31:39,800 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:31:39,802 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:49198, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:31:39,813 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-ns2-test-16900290656251-1690029097144/_SUCCESS 2023-07-22T12:31:39,819 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:31:39,821 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:42014, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:31:39,861 DEBUG [LoadIncrementalHFiles-0 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: d70ff17ae32b43b7898a7a0fa975c94f 2023-07-22T12:31:39,862 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-ns2-test-16900290656251-1690029097144/f/d70ff17ae32b43b7898a7a0fa975c94f first=Optional[row0] last=Optional[row98] 2023-07-22T12:31:39,887 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=ns2:test-16900290656251,,1690029096058.b8196e3466217e5359c3acacaaeb9e61., hostname=jenkins-hbase3.apache.org,43639,1690029056327, seqNum=2 for row with hfile group [{f,hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-ns2-test-16900290656251-1690029097144/f/d70ff17ae32b43b7898a7a0fa975c94f}] 2023-07-22T12:31:39,909 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-ns2-test-16900290656251-1690029097144/f/d70ff17ae32b43b7898a7a0fa975c94f for inclusion in b8196e3466217e5359c3acacaaeb9e61/f 2023-07-22T12:31:39,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-07-22T12:31:39,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HStore(625): Region bounds: first= last= 2023-07-22T12:31:39,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: d70ff17ae32b43b7898a7a0fa975c94f 2023-07-22T12:31:39,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HRegion(2520): Flush status journal for b8196e3466217e5359c3acacaaeb9e61: 2023-07-22T12:31:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-ns2-test-16900290656251-1690029097144/f/d70ff17ae32b43b7898a7a0fa975c94f to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/staging/jenkins__ns2_test-16900290656251__dedm4ai23re8sccn01lgucj4roig3bnmrgtca85r207uni1k1n5da3kl6f0fsku/f/d70ff17ae32b43b7898a7a0fa975c94f 2023-07-22T12:31:39,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/staging/jenkins__ns2_test-16900290656251__dedm4ai23re8sccn01lgucj4roig3bnmrgtca85r207uni1k1n5da3kl6f0fsku/f/d70ff17ae32b43b7898a7a0fa975c94f as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/b8196e3466217e5359c3acacaaeb9e61/f/82851155b50f47edb054b7e8d9f06090_SeqId_4_ 2023-07-22T12:31:39,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x389ea581 to 127.0.0.1:62657 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:31:39,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@210244db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:31:39,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:31:39,941 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:42026, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:31:39,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:31:39,948 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:49210, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:31:39,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] impl.BackupSystemTable(408): write bulk load descriptor to backup ns2:test-16900290656251 with 1 entries 2023-07-22T12:31:39,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] impl.BackupSystemTable(1582): writing raw bulk path hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/b8196e3466217e5359c3acacaaeb9e61/f/82851155b50f47edb054b7e8d9f06090_SeqId_4_ for ns2:test-16900290656251 b8196e3466217e5359c3acacaaeb9e61 2023-07-22T12:31:39,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] impl.BackupSystemTable(415): written 1 rows for bulk load of ns2:test-16900290656251 2023-07-22T12:31:39,980 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] client.ConnectionImplementation(2095): Closing master protocol: MasterService 2023-07-22T12:31:39,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x389ea581 to 127.0.0.1:62657 2023-07-22T12:31:39,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:31:39,981 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/staging/jenkins__ns2_test-16900290656251__dedm4ai23re8sccn01lgucj4roig3bnmrgtca85r207uni1k1n5da3kl6f0fsku/f/d70ff17ae32b43b7898a7a0fa975c94f into b8196e3466217e5359c3acacaaeb9e61/f as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/b8196e3466217e5359c3acacaaeb9e61/f/82851155b50f47edb054b7e8d9f06090_SeqId_4_ - updating store file list. 2023-07-22T12:31:39,989 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/b8196e3466217e5359c3acacaaeb9e61/f/82851155b50f47edb054b7e8d9f06090_SeqId_4_ into b8196e3466217e5359c3acacaaeb9e61/f 2023-07-22T12:31:39,989 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/staging/jenkins__ns2_test-16900290656251__dedm4ai23re8sccn01lgucj4roig3bnmrgtca85r207uni1k1n5da3kl6f0fsku/f/d70ff17ae32b43b7898a7a0fa975c94f into b8196e3466217e5359c3acacaaeb9e61/f (new location: hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/b8196e3466217e5359c3acacaaeb9e61/f/82851155b50f47edb054b7e8d9f06090_SeqId_4_) 2023-07-22T12:31:39,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/staging/jenkins__ns2_test-16900290656251__dedm4ai23re8sccn01lgucj4roig3bnmrgtca85r207uni1k1n5da3kl6f0fsku/f/d70ff17ae32b43b7898a7a0fa975c94f 2023-07-22T12:31:39,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0d79014e to 127.0.0.1:62657 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:31:40,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b775e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:31:40,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:31:40,007 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:42042, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-07-22T12:31:40,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:31:40,012 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:49216, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-07-22T12:31:40,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] impl.BackupSystemTable(388): write bulk load descriptor to backup ns2:test-16900290656251 with 1 entries 2023-07-22T12:31:40,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] impl.BackupSystemTable(1503): writing done bulk path hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/b8196e3466217e5359c3acacaaeb9e61/f/82851155b50f47edb054b7e8d9f06090_SeqId_4_ for ns2:test-16900290656251 b8196e3466217e5359c3acacaaeb9e61 2023-07-22T12:31:40,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] impl.BackupSystemTable(394): written 1 rows for bulk load of ns2:test-16900290656251 2023-07-22T12:31:40,036 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] client.ConnectionImplementation(2095): Closing master protocol: MasterService 2023-07-22T12:31:40,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0d79014e to 127.0.0.1:62657 2023-07-22T12:31:40,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:31:40,046 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] client.ConnectionImplementation(2095): Closing master protocol: MasterService 2023-07-22T12:31:40,046 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x4473ba0b to 127.0.0.1:62657 2023-07-22T12:31:40,047 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:31:40,047 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-07-22T12:31:40,048 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-07-22T12:31:40,048 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(233): Backup: backup_1690029074871 hdfs://localhost:34745/backupUT/backup_1690029074871/ns2/test-16900290656251/ 2023-07-22T12:31:40,049 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34745/backupUT/backup_1690029074871/.backup.manifest 2023-07-22T12:31:40,054 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029074871/.backup.manifest 2023-07-22T12:31:40,054 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(148): Restoring 'ns3:test-16900290656252' to 'ns3:test-16900290656252' from full backup image hdfs://localhost:34745/backupUT/backup_1690029074871/ns3/test-16900290656252 2023-07-22T12:31:40,063 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] util.RestoreTool(101): Folder tableArchivePath: hdfs://localhost:34745/backupUT/backup_1690029074871/ns3/test-16900290656252/archive/data/ns3/test-16900290656252 does not exists 2023-07-22T12:31:40,063 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] util.RestoreTool(326): find table descriptor but no archive dir for table ns3:test-16900290656252, will only create table 2023-07-22T12:31:40,064 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] util.RestoreTool(476): Truncating exising target table 'ns3:test-16900290656252', preserving region splits 2023-07-22T12:31:40,065 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] client.HBaseAdmin$18(963): Started disable of ns3:test-16900290656252 2023-07-22T12:31:40,066 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster$12(2663): Client=jenkins//172.31.12.81 disable ns3:test-16900290656252 2023-07-22T12:31:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=44, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=ns3:test-16900290656252 2023-07-22T12:31:40,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=44 2023-07-22T12:31:40,071 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16900290656252","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029100071"}]},"ts":"1690029100071"} 2023-07-22T12:31:40,073 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16900290656252, state=DISABLING in hbase:meta 2023-07-22T12:31:40,075 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(293): Set ns3:test-16900290656252 to state=DISABLING 2023-07-22T12:31:40,076 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns3:test-16900290656252, region=335c94783192a14614120d955d2aa716, UNASSIGN}] 2023-07-22T12:31:40,078 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=45, ppid=44, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns3:test-16900290656252, region=335c94783192a14614120d955d2aa716, UNASSIGN 2023-07-22T12:31:40,079 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=45 updating hbase:meta row=335c94783192a14614120d955d2aa716, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:40,080 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE; CloseRegionProcedure 335c94783192a14614120d955d2aa716, server=jenkins-hbase3.apache.org,43639,1690029056327}] 2023-07-22T12:31:40,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=44 2023-07-22T12:31:40,233 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] handler.UnassignRegionHandler(119): Close 335c94783192a14614120d955d2aa716 2023-07-22T12:31:40,233 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1668): Closing 335c94783192a14614120d955d2aa716, disabling compactions & flushes 2023-07-22T12:31:40,233 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1690): Closing region ns3:test-16900290656252,,1690029070575.335c94783192a14614120d955d2aa716. 2023-07-22T12:31:40,233 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16900290656252,,1690029070575.335c94783192a14614120d955d2aa716. 2023-07-22T12:31:40,233 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16900290656252,,1690029070575.335c94783192a14614120d955d2aa716. after waiting 0 ms 2023-07-22T12:31:40,233 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16900290656252,,1690029070575.335c94783192a14614120d955d2aa716. 2023-07-22T12:31:40,237 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns3/test-16900290656252/335c94783192a14614120d955d2aa716/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2023-07-22T12:31:40,238 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:31:40,239 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1909): Closed ns3:test-16900290656252,,1690029070575.335c94783192a14614120d955d2aa716. 2023-07-22T12:31:40,239 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1622): Region close journal for 335c94783192a14614120d955d2aa716: 2023-07-22T12:31:40,241 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] handler.UnassignRegionHandler(163): Closed 335c94783192a14614120d955d2aa716 2023-07-22T12:31:40,242 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=45 updating hbase:meta row=335c94783192a14614120d955d2aa716, regionState=CLOSED 2023-07-22T12:31:40,245 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=46, resume processing ppid=45 2023-07-22T12:31:40,245 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=46, ppid=45, state=SUCCESS; CloseRegionProcedure 335c94783192a14614120d955d2aa716, server=jenkins-hbase3.apache.org,43639,1690029056327 in 163 msec 2023-07-22T12:31:40,246 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=45, resume processing ppid=44 2023-07-22T12:31:40,247 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=45, ppid=44, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-16900290656252, region=335c94783192a14614120d955d2aa716, UNASSIGN in 169 msec 2023-07-22T12:31:40,247 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16900290656252","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029100247"}]},"ts":"1690029100247"} 2023-07-22T12:31:40,248 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16900290656252, state=DISABLED in hbase:meta 2023-07-22T12:31:40,250 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(305): Set ns3:test-16900290656252 to state=DISABLED 2023-07-22T12:31:40,252 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=44, state=SUCCESS; DisableTableProcedure table=ns3:test-16900290656252 in 185 msec 2023-07-22T12:31:40,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=44 2023-07-22T12:31:40,374 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] client.HBaseAdmin$TableFuture(3590): Operation: DISABLE, Table Name: ns3:test-16900290656252, procId: 44 completed 2023-07-22T12:31:40,374 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] client.HBaseAdmin$16(843): Started truncating ns3:test-16900290656252 2023-07-22T12:31:40,375 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster$6(2479): Client=jenkins//172.31.12.81 truncate ns3:test-16900290656252 2023-07-22T12:31:40,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=47, state=RUNNABLE:TRUNCATE_TABLE_PRE_OPERATION; TruncateTableProcedure (table=ns3:test-16900290656252 preserveSplits=true) 2023-07-22T12:31:40,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=47 2023-07-22T12:31:40,378 DEBUG [PEWorker-1 {}] procedure.TruncateTableProcedure(87): waiting for 'ns3:test-16900290656252' regions in transition 2023-07-22T12:31:40,381 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns3/test-16900290656252/335c94783192a14614120d955d2aa716 2023-07-22T12:31:40,383 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns3/test-16900290656252/335c94783192a14614120d955d2aa716/f, FileablePath, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns3/test-16900290656252/335c94783192a14614120d955d2aa716/recovered.edits] 2023-07-22T12:31:40,390 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns3/test-16900290656252/335c94783192a14614120d955d2aa716/recovered.edits/5.seqid to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/archive/data/ns3/test-16900290656252/335c94783192a14614120d955d2aa716/recovered.edits/5.seqid 2023-07-22T12:31:40,391 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns3/test-16900290656252/335c94783192a14614120d955d2aa716 2023-07-22T12:31:40,391 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(293): Archived ns3:test-16900290656252 regions 2023-07-22T12:31:40,397 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of ns3:test-16900290656252 from hbase:meta 2023-07-22T12:31:40,399 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(387): Removing 'ns3:test-16900290656252' descriptor. 2023-07-22T12:31:40,400 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(377): Removing 'ns3:test-16900290656252' from region states. 2023-07-22T12:31:40,400 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns3:test-16900290656252,,1690029070575.335c94783192a14614120d955d2aa716.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1690029100400"}]},"ts":"9223372036854775807"} 2023-07-22T12:31:40,402 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-07-22T12:31:40,402 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 335c94783192a14614120d955d2aa716, NAME => 'ns3:test-16900290656252,,1690029070575.335c94783192a14614120d955d2aa716.', STARTKEY => '', ENDKEY => ''}] 2023-07-22T12:31:40,402 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(381): Marking 'ns3:test-16900290656252' as deleted. 2023-07-22T12:31:40,402 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns3:test-16900290656252","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1690029100402"}]},"ts":"9223372036854775807"} 2023-07-22T12:31:40,405 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1678): Deleted table ns3:test-16900290656252 state from META 2023-07-22T12:31:40,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=47 2023-07-22T12:31:40,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=47 2023-07-22T12:31:40,816 INFO [RegionOpenAndInit-ns3:test-16900290656252-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => aefea80783ea312734eb5d79ad00b2f1, NAME => 'ns3:test-16900290656252,,1690029100393.aefea80783ea312734eb5d79ad00b2f1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-16900290656252', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947 2023-07-22T12:31:40,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=47 2023-07-22T12:31:41,224 DEBUG [RegionOpenAndInit-ns3:test-16900290656252-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:test-16900290656252,,1690029100393.aefea80783ea312734eb5d79ad00b2f1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:31:41,224 DEBUG [RegionOpenAndInit-ns3:test-16900290656252-pool-0 {}] regionserver.HRegion(1668): Closing aefea80783ea312734eb5d79ad00b2f1, disabling compactions & flushes 2023-07-22T12:31:41,224 INFO [RegionOpenAndInit-ns3:test-16900290656252-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:test-16900290656252,,1690029100393.aefea80783ea312734eb5d79ad00b2f1. 2023-07-22T12:31:41,224 DEBUG [RegionOpenAndInit-ns3:test-16900290656252-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16900290656252,,1690029100393.aefea80783ea312734eb5d79ad00b2f1. 2023-07-22T12:31:41,224 DEBUG [RegionOpenAndInit-ns3:test-16900290656252-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16900290656252,,1690029100393.aefea80783ea312734eb5d79ad00b2f1. after waiting 0 ms 2023-07-22T12:31:41,224 DEBUG [RegionOpenAndInit-ns3:test-16900290656252-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16900290656252,,1690029100393.aefea80783ea312734eb5d79ad00b2f1. 2023-07-22T12:31:41,224 INFO [RegionOpenAndInit-ns3:test-16900290656252-pool-0 {}] regionserver.HRegion(1909): Closed ns3:test-16900290656252,,1690029100393.aefea80783ea312734eb5d79ad00b2f1. 2023-07-22T12:31:41,224 DEBUG [RegionOpenAndInit-ns3:test-16900290656252-pool-0 {}] regionserver.HRegion(1622): Region close journal for aefea80783ea312734eb5d79ad00b2f1: 2023-07-22T12:31:41,225 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:test-16900290656252,,1690029100393.aefea80783ea312734eb5d79ad00b2f1.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1690029101225"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690029101225"}]},"ts":"1690029101225"} 2023-07-22T12:31:41,227 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:31:41,228 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16900290656252","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029101228"}]},"ts":"1690029101228"} 2023-07-22T12:31:41,230 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16900290656252, state=ENABLING in hbase:meta 2023-07-22T12:31:41,235 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=48, ppid=47, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16900290656252, region=aefea80783ea312734eb5d79ad00b2f1, ASSIGN}] 2023-07-22T12:31:41,237 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=48, ppid=47, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16900290656252, region=aefea80783ea312734eb5d79ad00b2f1, ASSIGN 2023-07-22T12:31:41,238 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=48, ppid=47, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:test-16900290656252, region=aefea80783ea312734eb5d79ad00b2f1, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,43639,1690029056327; forceNewPlan=false, retain=false 2023-07-22T12:31:41,390 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=48 updating hbase:meta row=aefea80783ea312734eb5d79ad00b2f1, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:41,392 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE; OpenRegionProcedure aefea80783ea312734eb5d79ad00b2f1, server=jenkins-hbase3.apache.org,43639,1690029056327}] 2023-07-22T12:31:41,439 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-07-22T12:31:41,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=47 2023-07-22T12:31:41,548 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(130): Open ns3:test-16900290656252,,1690029100393.aefea80783ea312734eb5d79ad00b2f1. 2023-07-22T12:31:41,548 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7254): Opening region: {ENCODED => aefea80783ea312734eb5d79ad00b2f1, NAME => 'ns3:test-16900290656252,,1690029100393.aefea80783ea312734eb5d79ad00b2f1.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:31:41,548 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:31:41,549 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16900290656252 aefea80783ea312734eb5d79ad00b2f1 2023-07-22T12:31:41,549 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(888): Instantiated ns3:test-16900290656252,,1690029100393.aefea80783ea312734eb5d79ad00b2f1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:31:41,549 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7296): checking encryption for aefea80783ea312734eb5d79ad00b2f1 2023-07-22T12:31:41,549 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7299): checking classloading for aefea80783ea312734eb5d79ad00b2f1 2023-07-22T12:31:41,550 INFO [StoreOpener-aefea80783ea312734eb5d79ad00b2f1-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region aefea80783ea312734eb5d79ad00b2f1 2023-07-22T12:31:41,552 INFO [StoreOpener-aefea80783ea312734eb5d79ad00b2f1-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aefea80783ea312734eb5d79ad00b2f1 columnFamilyName f 2023-07-22T12:31:41,552 DEBUG [StoreOpener-aefea80783ea312734eb5d79ad00b2f1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:31:41,553 INFO [StoreOpener-aefea80783ea312734eb5d79ad00b2f1-1 {}] regionserver.HStore(324): Store=aefea80783ea312734eb5d79ad00b2f1/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:31:41,554 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns3/test-16900290656252/aefea80783ea312734eb5d79ad00b2f1 2023-07-22T12:31:41,554 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns3/test-16900290656252/aefea80783ea312734eb5d79ad00b2f1 2023-07-22T12:31:41,556 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1076): writing seq id for aefea80783ea312734eb5d79ad00b2f1 2023-07-22T12:31:41,559 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns3/test-16900290656252/aefea80783ea312734eb5d79ad00b2f1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:31:41,559 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): Opened aefea80783ea312734eb5d79ad00b2f1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10942406880, jitterRate=0.019091054797172546}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-07-22T12:31:41,560 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(992): Region open journal for aefea80783ea312734eb5d79ad00b2f1: 2023-07-22T12:31:41,561 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2552): Post open deploy tasks for ns3:test-16900290656252,,1690029100393.aefea80783ea312734eb5d79ad00b2f1., pid=49, masterSystemTime=1690029101544 2023-07-22T12:31:41,563 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2579): Finished post open deploy task for ns3:test-16900290656252,,1690029100393.aefea80783ea312734eb5d79ad00b2f1. 2023-07-22T12:31:41,563 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(158): Opened ns3:test-16900290656252,,1690029100393.aefea80783ea312734eb5d79ad00b2f1. 2023-07-22T12:31:41,564 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=48 updating hbase:meta row=aefea80783ea312734eb5d79ad00b2f1, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:41,567 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=49, resume processing ppid=48 2023-07-22T12:31:41,568 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=49, ppid=48, state=SUCCESS; OpenRegionProcedure aefea80783ea312734eb5d79ad00b2f1, server=jenkins-hbase3.apache.org,43639,1690029056327 in 174 msec 2023-07-22T12:31:41,569 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=48, resume processing ppid=47 2023-07-22T12:31:41,569 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=48, ppid=47, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-16900290656252, region=aefea80783ea312734eb5d79ad00b2f1, ASSIGN in 332 msec 2023-07-22T12:31:41,569 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16900290656252","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029101569"}]},"ts":"1690029101569"} 2023-07-22T12:31:41,571 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16900290656252, state=ENABLED in hbase:meta 2023-07-22T12:31:41,575 DEBUG [PEWorker-2 {}] procedure.TruncateTableProcedure(145): truncate 'ns3:test-16900290656252' completed 2023-07-22T12:31:41,576 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=47, state=SUCCESS; TruncateTableProcedure (table=ns3:test-16900290656252 preserveSplits=true) in 1.2000 sec 2023-07-22T12:31:42,062 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2023-07-22T12:31:42,062 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2023-07-22T12:31:42,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=47 2023-07-22T12:31:42,485 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] client.HBaseAdmin$TableFuture(3590): Operation: TRUNCATE, Table Name: ns3:test-16900290656252, procId: 47 completed 2023-07-22T12:31:42,489 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-07-22T12:31:42,489 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(233): Backup: backup_1690029074871 hdfs://localhost:34745/backupUT/backup_1690029074871/ns3/test-16900290656252/ 2023-07-22T12:31:42,489 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-07-22T12:31:42,508 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreMultipleOverwrite Thread=598 (was 582) Potentially hanging thread: RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/cluster_3df5f3c2-318c-5fd6-9f69-2531a0a95eef/dfs/data/data2/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x4c100189-shared-pool-11 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_842990668_16 at /127.0.0.1:47376 [Waiting for operation #5] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x4c100189-shared-pool-12 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/cluster_3df5f3c2-318c-5fd6-9f69-2531a0a95eef/dfs/data/data1/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x4c100189-shared-pool-13 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_842990668_16 at /127.0.0.1:35084 [Receiving block BP-821173014-172.31.12.81-1690029049712:blk_1073741858_1034] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: HFileArchiver-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1319962190_16 at /127.0.0.1:35114 [Waiting for operation #37] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x4c100189-shared-pool-16 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: (jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-821173014-172.31.12.81-1690029049712:blk_1073741857_1033, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: HFileArchiver-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x4c100189-shared-pool-14 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_842990668_16 at /127.0.0.1:35074 [Receiving block BP-821173014-172.31.12.81-1690029049712:blk_1073741857_1033] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-821173014-172.31.12.81-1690029049712:blk_1073741858_1034, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x69245e7b-shared-pool-6 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x4c100189-shared-pool-15 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=672 (was 660) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=150 (was 156), ProcessCount=169 (was 169), AvailableMemoryMB=16907 (was 16702) - AvailableMemoryMB LEAK? - 2023-07-22T12:31:42,508 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(130): Thread=598 is superior to 500 2023-07-22T12:31:42,523 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreSingleOverwrite Thread=598, OpenFileDescriptor=672, MaxFileDescriptor=60000, SystemLoadAverage=150, ProcessCount=169, AvailableMemoryMB=16906 2023-07-22T12:31:42,523 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(130): Thread=598 is superior to 500 2023-07-22T12:31:42,523 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] backup.TestFullRestore(173): test full restore on a single table empty table 2023-07-22T12:31:42,524 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x2e82e540 to 127.0.0.1:62657 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:31:42,536 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2085bce3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:31:42,543 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:31:42,549 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:44528, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:31:42,551 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:31:42,553 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:52396, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:31:42,571 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-07-22T12:31:42,579 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:31:42,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029102579 (current time:1690029102579). 2023-07-22T12:31:42,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:31:42,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-07-22T12:31:42,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x72badf62 to 127.0.0.1:62657 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:31:42,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e4dbc21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:31:42,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:31:42,596 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:44542, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:31:42,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x72badf62 to 127.0.0.1:62657 2023-07-22T12:31:42,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:31:42,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:31:42,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=50, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:31:42,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 50 2023-07-22T12:31:42,601 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:31:42,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=50 2023-07-22T12:31:42,602 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:31:42,604 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:31:42,620 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:31:42,620 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33}] 2023-07-22T12:31:42,621 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=51, ppid=50, state=RUNNABLE; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33 2023-07-22T12:31:42,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=50 2023-07-22T12:31:42,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43639 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=51 2023-07-22T12:31:42,773 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. 2023-07-22T12:31:42,774 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegion(2819): Flushing 697cc5ecf0962f5ab9f4126febf7ce33 2/2 column families, dataSize=1.51 KB heapSize=2.81 KB 2023-07-22T12:31:42,793 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/meta/b580ac8bece04b07a21605e555bdc696 is 171, key is trslm:hdfs://localhost:34745/backupUT\x00ns2:test-16900290656251/meta:log-roll-map/1690029094729/Put/seqid=0 2023-07-22T12:31:42,799 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=792 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/meta/b580ac8bece04b07a21605e555bdc696 2023-07-22T12:31:42,805 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: b580ac8bece04b07a21605e555bdc696 2023-07-22T12:31:42,809 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/session/0704967b71ce4a5cb0af553e6e47addb is 458, key is session:backup_1690029074871/session:context/1690029095611/Put/seqid=0 2023-07-22T12:31:42,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=50 2023-07-22T12:31:43,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=50 2023-07-22T12:31:43,214 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=754 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/session/0704967b71ce4a5cb0af553e6e47addb 2023-07-22T12:31:43,221 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 0704967b71ce4a5cb0af553e6e47addb 2023-07-22T12:31:43,222 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/meta/b580ac8bece04b07a21605e555bdc696 as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/b580ac8bece04b07a21605e555bdc696 2023-07-22T12:31:43,229 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/b580ac8bece04b07a21605e555bdc696, entries=6, sequenceid=17, filesize=5.8 K 2023-07-22T12:31:43,230 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/session/0704967b71ce4a5cb0af553e6e47addb as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/0704967b71ce4a5cb0af553e6e47addb 2023-07-22T12:31:43,236 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/0704967b71ce4a5cb0af553e6e47addb, entries=2, sequenceid=17, filesize=5.4 K 2023-07-22T12:31:43,237 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegion(3022): Finished flush of dataSize ~1.51 KB/1546, heapSize ~2.78 KB/2848, currentSize=0 B/0 for 697cc5ecf0962f5ab9f4126febf7ce33 in 464ms, sequenceid=17, compaction requested=false 2023-07-22T12:31:43,237 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegion(2520): Flush status journal for 697cc5ecf0962f5ab9f4126febf7ce33: 2023-07-22T12:31:43,237 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. for snapshot_backup_system completed. 2023-07-22T12:31:43,238 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33.' region-info for snapshot=snapshot_backup_system 2023-07-22T12:31:43,238 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:31:43,238 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/b580ac8bece04b07a21605e555bdc696] hfiles 2023-07-22T12:31:43,238 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/b580ac8bece04b07a21605e555bdc696 for snapshot=snapshot_backup_system 2023-07-22T12:31:43,238 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/956010b3eaa64b4fa82ddea4557fac2c, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/0704967b71ce4a5cb0af553e6e47addb] hfiles 2023-07-22T12:31:43,238 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/956010b3eaa64b4fa82ddea4557fac2c for snapshot=snapshot_backup_system 2023-07-22T12:31:43,238 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/0704967b71ce4a5cb0af553e6e47addb for snapshot=snapshot_backup_system 2023-07-22T12:31:43,648 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. 2023-07-22T12:31:43,648 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=51 2023-07-22T12:31:43,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster(3994): Remote procedure done, pid=51 2023-07-22T12:31:43,649 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 697cc5ecf0962f5ab9f4126febf7ce33 2023-07-22T12:31:43,651 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=51, ppid=50, state=RUNNABLE; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33 2023-07-22T12:31:43,653 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=51, resume processing ppid=50 2023-07-22T12:31:43,653 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:31:43,653 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=51, ppid=50, state=SUCCESS; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33 in 1.0320 sec 2023-07-22T12:31:43,654 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:31:43,655 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:31:43,655 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-07-22T12:31:43,656 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_backup_system 2023-07-22T12:31:43,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=50 2023-07-22T12:31:44,074 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:31:44,081 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:31:44,082 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_backup_system 2023-07-22T12:31:44,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=50 2023-07-22T12:31:44,901 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:31:44,901 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 50 2023-07-22T12:31:44,902 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=50, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.3030 sec 2023-07-22T12:31:46,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=50 2023-07-22T12:31:46,711 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: backup:system, procId: 50 completed 2023-07-22T12:31:46,711 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(115): Backup backup_1690029102537 started at 1690029106711. 2023-07-22T12:31:46,713 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(119): Backup session backup_1690029102537 has been started. 2023-07-22T12:31:46,719 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-07-22T12:31:46,720 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-07-22T12:31:46,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-07-22T12:31:46,720 INFO [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-07-22T12:31:46,720 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-07-22T12:31:46,721 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-07-22T12:31:46,721 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-07-22T12:31:46,722 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-07-22T12:31:46,722 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:31:46,723 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:46,723 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): regionserver:43639-0x100005e31340001, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-07-22T12:31:46,723 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:31:46,723 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:31:46,723 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:46,723 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-07-22T12:31:46,724 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-07-22T12:31:46,724 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43639-0x100005e31340001, quorum=127.0.0.1:62657, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-07-22T12:31:46,724 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-07-22T12:31:46,724 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:31:46,724 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-07-22T12:31:46,725 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-07-22T12:31:46,725 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-07-22T12:31:46,725 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-07-22T12:31:46,725 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-07-22T12:31:46,726 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-07-22T12:31:46,726 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-07-22T12:31:46,726 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,43639,1690029056327' joining acquired barrier for procedure (rolllog) in zk 2023-07-22T12:31:46,728 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:46,728 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-07-22T12:31:46,729 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:46,729 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:31:46,729 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:31:46,729 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43639-0x100005e31340001, quorum=127.0.0.1:62657, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-07-22T12:31:46,729 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-07-22T12:31:46,729 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:31:46,729 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:31:46,730 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:31:46,730 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:46,730 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:31:46,731 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,43639,1690029056327' joining acquired barrier for procedure 'rolllog' on coordinator 2023-07-22T12:31:46,731 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@17476285[Count = 0] remaining members to acquire global barrier 2023-07-22T12:31:46,731 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-07-22T12:31:46,731 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-07-22T12:31:46,732 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): regionserver:43639-0x100005e31340001, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-07-22T12:31:46,732 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-07-22T12:31:46,732 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-07-22T12:31:46,732 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-07-22T12:31:46,732 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:46,732 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-07-22T12:31:46,732 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-07-22T12:31:46,732 DEBUG [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:46,732 INFO [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1690029082579 highest: 1690029082579 on jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:46,732 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta:.meta(num 1690029082559) roll requested 2023-07-22T12:31:46,744 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029106733.meta, exclude list is [], retry=0 2023-07-22T12:31:46,749 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK] 2023-07-22T12:31:46,751 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029082559.meta with entries=24, filesize=6.26 KB; new WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029106733.meta 2023-07-22T12:31:46,751 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK]] 2023-07-22T12:31:46,751 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029082559.meta is not closed yet, will try archiving it next time 2023-07-22T12:31:46,751 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C43639%2C1690029056327:(num 1690029082579) roll requested 2023-07-22T12:31:46,753 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029082559.meta not finished, retry = 0 2023-07-22T12:31:46,763 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029106751, exclude list is [], retry=0 2023-07-22T12:31:46,767 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK] 2023-07-22T12:31:46,768 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029082579 with entries=19, filesize=6.26 KB; new WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029106751 2023-07-22T12:31:46,768 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK]] 2023-07-22T12:31:46,768 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029082579 is not closed yet, will try archiving it next time 2023-07-22T12:31:46,773 DEBUG [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 41 2023-07-22T12:31:46,773 INFO [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1690029106751 on jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:46,783 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-07-22T12:31:46,783 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,43639,1690029056327' in zk 2023-07-22T12:31:46,785 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:46,785 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-07-22T12:31:46,785 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:31:46,785 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-07-22T12:31:46,785 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:46,786 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:31:46,786 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:31:46,786 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:31:46,787 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:31:46,787 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:31:46,787 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:46,787 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:31:46,788 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:31:46,788 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:46,789 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,43639,1690029056327': 2023-07-22T12:31:46,789 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,43639,1690029056327' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-07-22T12:31:46,789 INFO [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-07-22T12:31:46,789 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-07-22T12:31:46,789 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-07-22T12:31:46,789 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-07-22T12:31:46,789 INFO [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-07-22T12:31:46,791 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): regionserver:43639-0x100005e31340001, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-07-22T12:31:46,791 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-07-22T12:31:46,791 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): regionserver:43639-0x100005e31340001, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-07-22T12:31:46,791 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-07-22T12:31:46,791 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-07-22T12:31:46,791 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:31:46,791 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:31:46,791 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:31:46,791 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:31:46,791 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:31:46,791 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:31:46,791 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:46,792 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:31:46,792 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:31:46,792 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:31:46,793 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:31:46,793 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:46,793 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:46,793 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:31:46,794 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:31:46,794 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:46,798 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:46,798 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-07-22T12:31:46,798 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-07-22T12:31:46,798 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): regionserver:43639-0x100005e31340001, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-07-22T12:31:46,798 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:46,798 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-07-22T12:31:46,798 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): regionserver:43639-0x100005e31340001, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-07-22T12:31:46,798 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-07-22T12:31:46,798 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:31:46,798 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-07-22T12:31:46,799 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:31:46,799 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:31:46,799 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-07-22T12:31:46,799 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-07-22T12:31:46,800 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(347): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Unable to get data of znode /1/rolllog-proc/abort/rolllog because node does not exist (not an error) 2023-07-22T12:31:46,800 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:31:46,800 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:31:46,800 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] client.HBaseAdmin(2751): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-07-22T12:31:46,800 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] client.HBaseAdmin(2759): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-07-22T12:31:46,900 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] client.HBaseAdmin(2765): Getting current status of procedure from master... 2023-07-22T12:31:46,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-07-22T12:31:46,904 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1690029106903_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } 2023-07-22T12:31:46,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029106904 (current time:1690029106904). 2023-07-22T12:31:46,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:31:46,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1690029106903_default_test-1690029065625 VERSION not specified, setting to 2 2023-07-22T12:31:46,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x68cc5e0c to 127.0.0.1:62657 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:31:46,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b135b80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:31:46,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:31:46,913 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:44552, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:31:46,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x68cc5e0c to 127.0.0.1:62657 2023-07-22T12:31:46,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:31:46,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:31:46,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=52, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1690029106903_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } 2023-07-22T12:31:46,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1690029106903_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 }, snapshot procedure id = 52 2023-07-22T12:31:46,917 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1690029106903_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:31:46,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=52 2023-07-22T12:31:46,918 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1690029106903_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:31:46,920 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1690029106903_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:31:47,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=52 2023-07-22T12:31:47,051 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-07-22T12:31:47,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=52 2023-07-22T12:31:47,334 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1690029106903_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:31:47,334 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE; SnapshotRegionProcedure 1a5339f2a886ed4b01c0668fe0d77c7f}] 2023-07-22T12:31:47,337 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=53, ppid=52, state=RUNNABLE; SnapshotRegionProcedure 1a5339f2a886ed4b01c0668fe0d77c7f 2023-07-22T12:31:47,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43639 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=53 2023-07-22T12:31:47,489 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1690029065625,,1690029066110.1a5339f2a886ed4b01c0668fe0d77c7f. 2023-07-22T12:31:47,490 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(2819): Flushing 1a5339f2a886ed4b01c0668fe0d77c7f 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-07-22T12:31:47,504 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-16900290656252' 2023-07-22T12:31:47,512 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/1a5339f2a886ed4b01c0668fe0d77c7f/.tmp/f/b2ab9a7120ba4859a7efb56e7d3679e8 is 37, key is row10/f:q1/1690029068268/Put/seqid=0 2023-07-22T12:31:47,517 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/1a5339f2a886ed4b01c0668fe0d77c7f/.tmp/f/b2ab9a7120ba4859a7efb56e7d3679e8 2023-07-22T12:31:47,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=52 2023-07-22T12:31:47,523 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: b2ab9a7120ba4859a7efb56e7d3679e8 2023-07-22T12:31:47,524 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/1a5339f2a886ed4b01c0668fe0d77c7f/.tmp/f/b2ab9a7120ba4859a7efb56e7d3679e8 as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/1a5339f2a886ed4b01c0668fe0d77c7f/f/b2ab9a7120ba4859a7efb56e7d3679e8 2023-07-22T12:31:47,531 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/1a5339f2a886ed4b01c0668fe0d77c7f/f/b2ab9a7120ba4859a7efb56e7d3679e8, entries=99, sequenceid=103, filesize=8.2 K 2023-07-22T12:31:47,532 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 1a5339f2a886ed4b01c0668fe0d77c7f in 42ms, sequenceid=103, compaction requested=false 2023-07-22T12:31:47,533 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(2520): Flush status journal for 1a5339f2a886ed4b01c0668fe0d77c7f: 2023-07-22T12:31:47,533 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1690029065625,,1690029066110.1a5339f2a886ed4b01c0668fe0d77c7f. for snapshot_1690029106903_default_test-1690029065625 completed. 2023-07-22T12:31:47,533 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(240): Storing 'test-1690029065625,,1690029066110.1a5339f2a886ed4b01c0668fe0d77c7f.' region-info for snapshot=snapshot_1690029106903_default_test-1690029065625 2023-07-22T12:31:47,533 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:31:47,533 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/1a5339f2a886ed4b01c0668fe0d77c7f/f/b2ab9a7120ba4859a7efb56e7d3679e8] hfiles 2023-07-22T12:31:47,533 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/1a5339f2a886ed4b01c0668fe0d77c7f/f/b2ab9a7120ba4859a7efb56e7d3679e8 for snapshot=snapshot_1690029106903_default_test-1690029065625 2023-07-22T12:31:47,939 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1690029065625,,1690029066110.1a5339f2a886ed4b01c0668fe0d77c7f. 2023-07-22T12:31:47,939 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=53 2023-07-22T12:31:47,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster(3994): Remote procedure done, pid=53 2023-07-22T12:31:47,940 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1690029106903_default_test-1690029065625 on region 1a5339f2a886ed4b01c0668fe0d77c7f 2023-07-22T12:31:47,941 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=53, ppid=52, state=RUNNABLE; SnapshotRegionProcedure 1a5339f2a886ed4b01c0668fe0d77c7f 2023-07-22T12:31:47,944 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=53, resume processing ppid=52 2023-07-22T12:31:47,944 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1690029106903_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:31:47,944 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=53, ppid=52, state=SUCCESS; SnapshotRegionProcedure 1a5339f2a886ed4b01c0668fe0d77c7f in 608 msec 2023-07-22T12:31:47,945 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1690029106903_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:31:47,945 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1690029106903_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:31:47,945 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1690029106903_default_test-1690029065625 2023-07-22T12:31:47,946 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_1690029106903_default_test-1690029065625 2023-07-22T12:31:48,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=52 2023-07-22T12:31:48,357 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1690029106903_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:31:48,366 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1690029106903_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:31:48,367 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_1690029106903_default_test-1690029065625 to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029106903_default_test-1690029065625 2023-07-22T12:31:48,389 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1690029106903_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:31:48,389 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1690029106903_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 }, snapshot procedure id = 52 2023-07-22T12:31:48,390 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=52, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1690029106903_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } in 1.4740 sec 2023-07-22T12:31:49,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=52 2023-07-22T12:31:49,026 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: default:test-1690029065625, procId: 52 completed 2023-07-22T12:31:49,026 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1690029102537 2023-07-22T12:31:49,026 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-07-22T12:31:49,026 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-07-22T12:31:49,026 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1690029102537_test-1690029065625 2023-07-22T12:31:49,026 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1690029106903_default_test-1690029065625 to hdfs://localhost:34745/backupUT/backup_1690029102537/default/test-1690029065625/ 2023-07-22T12:31:49,026 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-07-22T12:31:49,065 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:34745, inputRoot=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947 2023-07-22T12:31:49,065 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,37923,1690029053050_-1155959680_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:34745/backupUT/backup_1690029102537/default/test-1690029065625, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:34745/backupUT/backup_1690029102537/default/test-1690029065625/.hbase-snapshot/.tmp/snapshot_1690029106903_default_test-1690029065625 2023-07-22T12:31:49,065 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:34745, inputRoot=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947, snapshotDir=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029106903_default_test-1690029065625. 2023-07-22T12:31:49,078 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029106903_default_test-1690029065625 to hdfs://localhost:34745/backupUT/backup_1690029102537/default/test-1690029065625/.hbase-snapshot/.tmp/snapshot_1690029106903_default_test-1690029065625 2023-07-22T12:31:49,114 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:49,114 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:49,115 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:49,116 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:50,121 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/hadoop-6300359206115901200.jar 2023-07-22T12:31:50,122 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:50,122 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:50,123 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:50,123 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:50,124 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:50,124 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:50,124 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:50,125 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:50,125 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:50,126 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:31:50,127 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-07-22T12:31:50,127 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-07-22T12:31:50,127 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-07-22T12:31:50,128 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-07-22T12:31:50,128 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-07-22T12:31:50,129 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-07-22T12:31:50,129 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-07-22T12:31:50,129 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-07-22T12:31:50,130 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-07-22T12:31:50,130 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-07-22T12:31:50,131 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-07-22T12:31:50,131 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:31:50,132 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:31:50,132 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:31:50,133 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:31:50,133 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:31:50,134 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:31:50,134 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:31:50,162 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-07-22T12:31:50,180 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1690029106903_default_test-1690029065625' hfile list 2023-07-22T12:31:50,182 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-07-22T12:31:50,255 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029110220/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-07-22T12:31:50,255 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029110220/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-07-22T12:31:50,312 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-07-22T12:31:50,362 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/1a5339f2a886ed4b01c0668fe0d77c7f/f/b2ab9a7120ba4859a7efb56e7d3679e8 output=hdfs://localhost:34745/backupUT/backup_1690029102537/default/test-1690029065625/archive/data/default/test-1690029065625/1a5339f2a886ed4b01c0668fe0d77c7f/f/b2ab9a7120ba4859a7efb56e7d3679e8 2023-07-22T12:31:50,362 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 8.007M/sec 2023-07-22T12:31:51,293 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-07-22T12:31:51,294 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-07-22T12:31:51,301 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1690029106903_default_test-1690029065625 2023-07-22T12:31:51,301 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1690029106903_default_test-1690029065625 finished. 2023-07-22T12:31:51,301 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(1014): test-1690029065625 2023-07-22T12:31:51,308 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029102537 2023-07-22T12:31:51,308 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-07-22T12:31:51,715 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:34745/backupUT/backup_1690029102537/.backup.manifest 2023-07-22T12:31:51,715 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(358): Backup backup_1690029102537 finished: type=FULL,tablelist=test-1690029065625,targetRootDir=hdfs://localhost:34745/backupUT,startts=1690029106711,completets=1690029111308,bytescopied=0 2023-07-22T12:31:51,715 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-07-22T12:31:51,715 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1690029106903_default_test-1690029065625 2023-07-22T12:31:51,716 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1690029106903_default_test-1690029065625" 2023-07-22T12:31:51,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1690029106903_default_test-1690029065625 2023-07-22T12:31:51,720 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1690029106903_default_test-1690029065625 for backup backup_1690029102537 succeeded. 2023-07-22T12:31:51,720 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-07-22T12:31:51,725 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-07-22T12:31:51,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-07-22T12:31:51,729 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-07-22T12:31:51,731 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-07-22T12:31:51,737 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(376): Backup backup_1690029102537 completed. 2023-07-22T12:31:51,737 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] client.ConnectionImplementation(2095): Closing master protocol: MasterService 2023-07-22T12:31:51,737 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x2e82e540 to 127.0.0.1:62657 2023-07-22T12:31:51,737 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:31:51,752 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] backup.TestFullRestore(178): backup complete 2023-07-22T12:31:51,755 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34745/backupUT/backup_1690029102537/.backup.manifest 2023-07-22T12:31:51,757 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029102537/.backup.manifest 2023-07-22T12:31:51,758 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34745/backupUT/backup_1690029102537/.backup.manifest 2023-07-22T12:31:51,760 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029102537/.backup.manifest 2023-07-22T12:31:51,760 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(148): Restoring 'test-1690029065625' to 'test-1690029065625' from full backup image hdfs://localhost:34745/backupUT/backup_1690029102537/default/test-1690029065625 2023-07-22T12:31:51,768 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] util.RestoreTool(476): Truncating exising target table 'test-1690029065625', preserving region splits 2023-07-22T12:31:51,769 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] client.HBaseAdmin$18(963): Started disable of test-1690029065625 2023-07-22T12:31:51,769 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster$12(2663): Client=jenkins//172.31.12.81 disable test-1690029065625 2023-07-22T12:31:51,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=test-1690029065625 2023-07-22T12:31:51,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=54 2023-07-22T12:31:51,773 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1690029065625","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029111773"}]},"ts":"1690029111773"} 2023-07-22T12:31:51,775 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1690029065625, state=DISABLING in hbase:meta 2023-07-22T12:31:51,777 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(293): Set test-1690029065625 to state=DISABLING 2023-07-22T12:31:51,778 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=test-1690029065625, region=1a5339f2a886ed4b01c0668fe0d77c7f, UNASSIGN}] 2023-07-22T12:31:51,779 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=test-1690029065625, region=1a5339f2a886ed4b01c0668fe0d77c7f, UNASSIGN 2023-07-22T12:31:51,780 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=1a5339f2a886ed4b01c0668fe0d77c7f, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:51,781 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE; CloseRegionProcedure 1a5339f2a886ed4b01c0668fe0d77c7f, server=jenkins-hbase3.apache.org,43639,1690029056327}] 2023-07-22T12:31:51,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=54 2023-07-22T12:31:51,933 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] handler.UnassignRegionHandler(119): Close 1a5339f2a886ed4b01c0668fe0d77c7f 2023-07-22T12:31:51,933 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1668): Closing 1a5339f2a886ed4b01c0668fe0d77c7f, disabling compactions & flushes 2023-07-22T12:31:51,933 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1690): Closing region test-1690029065625,,1690029066110.1a5339f2a886ed4b01c0668fe0d77c7f. 2023-07-22T12:31:51,933 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1690029065625,,1690029066110.1a5339f2a886ed4b01c0668fe0d77c7f. 2023-07-22T12:31:51,933 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1778): Acquired close lock on test-1690029065625,,1690029066110.1a5339f2a886ed4b01c0668fe0d77c7f. after waiting 0 ms 2023-07-22T12:31:51,933 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1788): Updates disabled for region test-1690029065625,,1690029066110.1a5339f2a886ed4b01c0668fe0d77c7f. 2023-07-22T12:31:51,935 DEBUG [StoreFileCloser-1a5339f2a886ed4b01c0668fe0d77c7f-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: b2ab9a7120ba4859a7efb56e7d3679e8 2023-07-22T12:31:51,940 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/1a5339f2a886ed4b01c0668fe0d77c7f/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2023-07-22T12:31:51,940 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:31:51,941 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1909): Closed test-1690029065625,,1690029066110.1a5339f2a886ed4b01c0668fe0d77c7f. 2023-07-22T12:31:51,941 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1622): Region close journal for 1a5339f2a886ed4b01c0668fe0d77c7f: 2023-07-22T12:31:51,943 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] handler.UnassignRegionHandler(163): Closed 1a5339f2a886ed4b01c0668fe0d77c7f 2023-07-22T12:31:51,943 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=1a5339f2a886ed4b01c0668fe0d77c7f, regionState=CLOSED 2023-07-22T12:31:51,946 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=56, resume processing ppid=55 2023-07-22T12:31:51,946 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=56, ppid=55, state=SUCCESS; CloseRegionProcedure 1a5339f2a886ed4b01c0668fe0d77c7f, server=jenkins-hbase3.apache.org,43639,1690029056327 in 163 msec 2023-07-22T12:31:51,947 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=55, resume processing ppid=54 2023-07-22T12:31:51,947 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=55, ppid=54, state=SUCCESS; TransitRegionStateProcedure table=test-1690029065625, region=1a5339f2a886ed4b01c0668fe0d77c7f, UNASSIGN in 168 msec 2023-07-22T12:31:51,948 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1690029065625","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029111948"}]},"ts":"1690029111948"} 2023-07-22T12:31:51,949 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1690029065625, state=DISABLED in hbase:meta 2023-07-22T12:31:51,951 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(305): Set test-1690029065625 to state=DISABLED 2023-07-22T12:31:51,952 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=54, state=SUCCESS; DisableTableProcedure table=test-1690029065625 in 181 msec 2023-07-22T12:31:52,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=54 2023-07-22T12:31:52,076 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] client.HBaseAdmin$TableFuture(3590): Operation: DISABLE, Table Name: default:test-1690029065625, procId: 54 completed 2023-07-22T12:31:52,076 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] client.HBaseAdmin$16(843): Started truncating test-1690029065625 2023-07-22T12:31:52,076 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster$6(2479): Client=jenkins//172.31.12.81 truncate test-1690029065625 2023-07-22T12:31:52,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=57, state=RUNNABLE:TRUNCATE_TABLE_PRE_OPERATION; TruncateTableProcedure (table=test-1690029065625 preserveSplits=true) 2023-07-22T12:31:52,079 DEBUG [PEWorker-2 {}] procedure.TruncateTableProcedure(87): waiting for 'test-1690029065625' regions in transition 2023-07-22T12:31:52,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=57 2023-07-22T12:31:52,082 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/1a5339f2a886ed4b01c0668fe0d77c7f 2023-07-22T12:31:52,085 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/1a5339f2a886ed4b01c0668fe0d77c7f/f, FileablePath, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/1a5339f2a886ed4b01c0668fe0d77c7f/recovered.edits] 2023-07-22T12:31:52,089 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/1a5339f2a886ed4b01c0668fe0d77c7f/f/b2ab9a7120ba4859a7efb56e7d3679e8 to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/archive/data/default/test-1690029065625/1a5339f2a886ed4b01c0668fe0d77c7f/f/b2ab9a7120ba4859a7efb56e7d3679e8 2023-07-22T12:31:52,092 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/1a5339f2a886ed4b01c0668fe0d77c7f/recovered.edits/106.seqid to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/archive/data/default/test-1690029065625/1a5339f2a886ed4b01c0668fe0d77c7f/recovered.edits/106.seqid 2023-07-22T12:31:52,092 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/1a5339f2a886ed4b01c0668fe0d77c7f 2023-07-22T12:31:52,092 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(293): Archived test-1690029065625 regions 2023-07-22T12:31:52,096 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of test-1690029065625 from hbase:meta 2023-07-22T12:31:52,098 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(387): Removing 'test-1690029065625' descriptor. 2023-07-22T12:31:52,098 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(377): Removing 'test-1690029065625' from region states. 2023-07-22T12:31:52,099 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"test-1690029065625,,1690029066110.1a5339f2a886ed4b01c0668fe0d77c7f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1690029112098"}]},"ts":"9223372036854775807"} 2023-07-22T12:31:52,100 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-07-22T12:31:52,100 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 1a5339f2a886ed4b01c0668fe0d77c7f, NAME => 'test-1690029065625,,1690029066110.1a5339f2a886ed4b01c0668fe0d77c7f.', STARTKEY => '', ENDKEY => ''}] 2023-07-22T12:31:52,100 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(381): Marking 'test-1690029065625' as deleted. 2023-07-22T12:31:52,100 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"test-1690029065625","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1690029112100"}]},"ts":"9223372036854775807"} 2023-07-22T12:31:52,102 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1678): Deleted table test-1690029065625 state from META 2023-07-22T12:31:52,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=57 2023-07-22T12:31:52,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=57 2023-07-22T12:31:52,512 INFO [RegionOpenAndInit-test-1690029065625-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => d1d0c6c0a475cf01e7de961f9001d124, NAME => 'test-1690029065625,,1690029112094.d1d0c6c0a475cf01e7de961f9001d124.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1690029065625', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947 2023-07-22T12:31:52,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=57 2023-07-22T12:31:52,920 DEBUG [RegionOpenAndInit-test-1690029065625-pool-0 {}] regionserver.HRegion(888): Instantiated test-1690029065625,,1690029112094.d1d0c6c0a475cf01e7de961f9001d124.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:31:52,920 DEBUG [RegionOpenAndInit-test-1690029065625-pool-0 {}] regionserver.HRegion(1668): Closing d1d0c6c0a475cf01e7de961f9001d124, disabling compactions & flushes 2023-07-22T12:31:52,920 INFO [RegionOpenAndInit-test-1690029065625-pool-0 {}] regionserver.HRegion(1690): Closing region test-1690029065625,,1690029112094.d1d0c6c0a475cf01e7de961f9001d124. 2023-07-22T12:31:52,920 DEBUG [RegionOpenAndInit-test-1690029065625-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1690029065625,,1690029112094.d1d0c6c0a475cf01e7de961f9001d124. 2023-07-22T12:31:52,920 DEBUG [RegionOpenAndInit-test-1690029065625-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on test-1690029065625,,1690029112094.d1d0c6c0a475cf01e7de961f9001d124. after waiting 0 ms 2023-07-22T12:31:52,920 DEBUG [RegionOpenAndInit-test-1690029065625-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region test-1690029065625,,1690029112094.d1d0c6c0a475cf01e7de961f9001d124. 2023-07-22T12:31:52,920 INFO [RegionOpenAndInit-test-1690029065625-pool-0 {}] regionserver.HRegion(1909): Closed test-1690029065625,,1690029112094.d1d0c6c0a475cf01e7de961f9001d124. 2023-07-22T12:31:52,920 DEBUG [RegionOpenAndInit-test-1690029065625-pool-0 {}] regionserver.HRegion(1622): Region close journal for d1d0c6c0a475cf01e7de961f9001d124: 2023-07-22T12:31:52,922 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"test-1690029065625,,1690029112094.d1d0c6c0a475cf01e7de961f9001d124.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1690029112921"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690029112921"}]},"ts":"1690029112921"} 2023-07-22T12:31:52,923 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:31:52,924 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1690029065625","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029112924"}]},"ts":"1690029112924"} 2023-07-22T12:31:52,925 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1690029065625, state=ENABLING in hbase:meta 2023-07-22T12:31:52,930 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1690029065625, region=d1d0c6c0a475cf01e7de961f9001d124, ASSIGN}] 2023-07-22T12:31:52,931 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1690029065625, region=d1d0c6c0a475cf01e7de961f9001d124, ASSIGN 2023-07-22T12:31:52,932 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=test-1690029065625, region=d1d0c6c0a475cf01e7de961f9001d124, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,43639,1690029056327; forceNewPlan=false, retain=false 2023-07-22T12:31:53,084 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=58 updating hbase:meta row=d1d0c6c0a475cf01e7de961f9001d124, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:53,085 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE; OpenRegionProcedure d1d0c6c0a475cf01e7de961f9001d124, server=jenkins-hbase3.apache.org,43639,1690029056327}] 2023-07-22T12:31:53,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=57 2023-07-22T12:31:53,225 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1690029065625 2023-07-22T12:31:53,240 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(130): Open test-1690029065625,,1690029112094.d1d0c6c0a475cf01e7de961f9001d124. 2023-07-22T12:31:53,241 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7254): Opening region: {ENCODED => d1d0c6c0a475cf01e7de961f9001d124, NAME => 'test-1690029065625,,1690029112094.d1d0c6c0a475cf01e7de961f9001d124.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:31:53,241 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:31:53,241 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1690029065625 d1d0c6c0a475cf01e7de961f9001d124 2023-07-22T12:31:53,241 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(888): Instantiated test-1690029065625,,1690029112094.d1d0c6c0a475cf01e7de961f9001d124.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:31:53,241 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7296): checking encryption for d1d0c6c0a475cf01e7de961f9001d124 2023-07-22T12:31:53,241 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7299): checking classloading for d1d0c6c0a475cf01e7de961f9001d124 2023-07-22T12:31:53,242 INFO [StoreOpener-d1d0c6c0a475cf01e7de961f9001d124-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region d1d0c6c0a475cf01e7de961f9001d124 2023-07-22T12:31:53,244 INFO [StoreOpener-d1d0c6c0a475cf01e7de961f9001d124-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1d0c6c0a475cf01e7de961f9001d124 columnFamilyName f 2023-07-22T12:31:53,244 DEBUG [StoreOpener-d1d0c6c0a475cf01e7de961f9001d124-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:31:53,244 INFO [StoreOpener-d1d0c6c0a475cf01e7de961f9001d124-1 {}] regionserver.HStore(324): Store=d1d0c6c0a475cf01e7de961f9001d124/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:31:53,245 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/d1d0c6c0a475cf01e7de961f9001d124 2023-07-22T12:31:53,245 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/d1d0c6c0a475cf01e7de961f9001d124 2023-07-22T12:31:53,247 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1076): writing seq id for d1d0c6c0a475cf01e7de961f9001d124 2023-07-22T12:31:53,249 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/d1d0c6c0a475cf01e7de961f9001d124/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:31:53,249 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): Opened d1d0c6c0a475cf01e7de961f9001d124; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10094790240, jitterRate=-0.05984939634799957}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-07-22T12:31:53,250 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(992): Region open journal for d1d0c6c0a475cf01e7de961f9001d124: 2023-07-22T12:31:53,250 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2552): Post open deploy tasks for test-1690029065625,,1690029112094.d1d0c6c0a475cf01e7de961f9001d124., pid=59, masterSystemTime=1690029113237 2023-07-22T12:31:53,252 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2579): Finished post open deploy task for test-1690029065625,,1690029112094.d1d0c6c0a475cf01e7de961f9001d124. 2023-07-22T12:31:53,252 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(158): Opened test-1690029065625,,1690029112094.d1d0c6c0a475cf01e7de961f9001d124. 2023-07-22T12:31:53,252 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=58 updating hbase:meta row=d1d0c6c0a475cf01e7de961f9001d124, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:31:53,256 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=59, resume processing ppid=58 2023-07-22T12:31:53,256 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=59, ppid=58, state=SUCCESS; OpenRegionProcedure d1d0c6c0a475cf01e7de961f9001d124, server=jenkins-hbase3.apache.org,43639,1690029056327 in 169 msec 2023-07-22T12:31:53,258 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=58, resume processing ppid=57 2023-07-22T12:31:53,258 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=58, ppid=57, state=SUCCESS; TransitRegionStateProcedure table=test-1690029065625, region=d1d0c6c0a475cf01e7de961f9001d124, ASSIGN in 326 msec 2023-07-22T12:31:53,258 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1690029065625","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029113258"}]},"ts":"1690029113258"} 2023-07-22T12:31:53,260 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1690029065625, state=ENABLED in hbase:meta 2023-07-22T12:31:53,261 DEBUG [PEWorker-3 {}] procedure.TruncateTableProcedure(145): truncate 'test-1690029065625' completed 2023-07-22T12:31:53,263 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=57, state=SUCCESS; TruncateTableProcedure (table=test-1690029065625 preserveSplits=true) in 1.1850 sec 2023-07-22T12:31:54,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=57 2023-07-22T12:31:54,186 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] client.HBaseAdmin$TableFuture(3590): Operation: TRUNCATE, Table Name: default:test-1690029065625, procId: 57 completed 2023-07-22T12:31:54,190 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:34745/backupUT/backup_1690029102537/default/test-1690029065625/archive/data/default/test-1690029065625/1a5339f2a886ed4b01c0668fe0d77c7f from hbase tables test-1690029065625 to tables test-1690029065625 2023-07-22T12:31:54,190 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceRestoreJob(72): Restore test-1690029065625 into test-1690029065625 2023-07-22T12:31:54,192 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-default-test-1690029065625-1690029114190 from hdfs://localhost:34745/backupUT/backup_1690029102537/default/test-1690029065625/archive/data/default/test-1690029065625/1a5339f2a886ed4b01c0668fe0d77c7f 2023-07-22T12:31:54,193 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x608095eb to 127.0.0.1:62657 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:31:54,199 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d3a809, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:31:54,200 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:31:54,202 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:57614, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:31:54,203 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-07-22T12:31:54,203 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table test-1690029065625 2023-07-22T12:31:54,206 DEBUG [hconnection-0x32ec3d84-shared-pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:31:54,208 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:39220, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:31:54,209 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for test-1690029065625: 2023-07-22T12:31:54,209 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-07-22T12:31:54,209 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/partitions_5be7ad9b-85a6-40ab-be23-7725fa89175b 2023-07-22T12:31:54,619 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:54,620 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:54,620 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:54,621 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:55,582 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/hadoop-6925328429872797260.jar 2023-07-22T12:31:55,582 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:55,583 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:55,583 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:55,584 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:55,584 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:55,585 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:55,585 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:55,586 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:55,586 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:55,587 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:31:55,587 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-07-22T12:31:55,588 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-07-22T12:31:55,588 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-07-22T12:31:55,589 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-07-22T12:31:55,589 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-07-22T12:31:55,590 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-07-22T12:31:55,590 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-07-22T12:31:55,590 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-07-22T12:31:55,591 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-07-22T12:31:55,591 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-07-22T12:31:55,591 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-07-22T12:31:55,592 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:55,592 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:55,593 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:55,593 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:55,593 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:55,594 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:31:55,594 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:31:55,595 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: test-1690029065625 2023-07-22T12:31:55,595 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] client.ConnectionImplementation(2095): Closing master protocol: MasterService 2023-07-22T12:31:55,595 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x608095eb to 127.0.0.1:62657 2023-07-22T12:31:55,595 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:31:55,595 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-07-22T12:31:55,595 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-07-22T12:31:55,596 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:31:55,601 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-07-22T12:31:55,606 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-07-22T12:31:55,680 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029115647/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-07-22T12:31:55,680 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029115647/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-07-22T12:31:55,741 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:34745/backupUT/backup_1690029102537/default/test-1690029065625/archive/data/default/test-1690029065625/1a5339f2a886ed4b01c0668fe0d77c7f/f/b2ab9a7120ba4859a7efb56e7d3679e8 2023-07-22T12:31:55,746 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-07-22T12:31:55,749 DEBUG [LocalJobRunner Map Task Executor #0 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: b2ab9a7120ba4859a7efb56e7d3679e8 2023-07-22T12:31:55,765 DEBUG [pool-234-thread-1 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x3db5c00b to 127.0.0.1:62657 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:31:55,770 DEBUG [pool-234-thread-1 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a7842c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:31:55,771 DEBUG [hconnection-0x6c5cb77f-metaLookup-shared--pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:31:55,773 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:39222, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:31:55,777 DEBUG [pool-234-thread-1 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x3db5c00b to 127.0.0.1:62657 2023-07-22T12:31:55,777 DEBUG [pool-234-thread-1 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:31:55,777 DEBUG [pool-234-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-07-22T12:31:55,777 DEBUG [pool-234-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-07-22T12:31:55,785 DEBUG [pool-234-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-default-test-1690029065625-1690029114190/_temporary/0/_temporary/attempt_local308600813_0005_r_000000_0/f/79a99e90c2c84e3cb4186b5fa29d3371 is 37, key is row10/f:q1/1690029068268/Put/seqid=0 2023-07-22T12:31:55,786 WARN [Thread-992 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:43639] Specified favored nodes: [jenkins-hbase3.apache.org:43639] 2023-07-22T12:31:56,741 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-default-test-1690029065625-1690029114190 2023-07-22T12:31:56,741 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x2a38abef to 127.0.0.1:62657 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:31:56,749 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a6362f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:31:56,751 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:31:56,752 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:39236, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:31:56,759 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-default-test-1690029065625-1690029114190/_SUCCESS 2023-07-22T12:31:56,763 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:31:56,764 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:57620, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:31:56,774 DEBUG [LoadIncrementalHFiles-0 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 79a99e90c2c84e3cb4186b5fa29d3371 2023-07-22T12:31:56,774 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-default-test-1690029065625-1690029114190/f/79a99e90c2c84e3cb4186b5fa29d3371 first=Optional[row0] last=Optional[row98] 2023-07-22T12:31:56,775 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=test-1690029065625,,1690029112094.d1d0c6c0a475cf01e7de961f9001d124., hostname=jenkins-hbase3.apache.org,43639,1690029056327, seqNum=2 for row with hfile group [{f,hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-default-test-1690029065625-1690029114190/f/79a99e90c2c84e3cb4186b5fa29d3371}] 2023-07-22T12:31:56,781 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-default-test-1690029065625-1690029114190/f/79a99e90c2c84e3cb4186b5fa29d3371 for inclusion in d1d0c6c0a475cf01e7de961f9001d124/f 2023-07-22T12:31:56,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-07-22T12:31:56,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HStore(625): Region bounds: first= last= 2023-07-22T12:31:56,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 79a99e90c2c84e3cb4186b5fa29d3371 2023-07-22T12:31:56,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HRegion(2520): Flush status journal for d1d0c6c0a475cf01e7de961f9001d124: 2023-07-22T12:31:56,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-default-test-1690029065625-1690029114190/f/79a99e90c2c84e3cb4186b5fa29d3371 to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/staging/jenkins__test-1690029065625__edaj3f48mta1udc6nk152sb3222a80976hc9ve7tnbg92va6ocdtchvs74n584qo/f/79a99e90c2c84e3cb4186b5fa29d3371 2023-07-22T12:31:56,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/staging/jenkins__test-1690029065625__edaj3f48mta1udc6nk152sb3222a80976hc9ve7tnbg92va6ocdtchvs74n584qo/f/79a99e90c2c84e3cb4186b5fa29d3371 as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/d1d0c6c0a475cf01e7de961f9001d124/f/acd648b54cf0414a8319c9fc8c54d368_SeqId_4_ 2023-07-22T12:31:56,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x6eecc919 to 127.0.0.1:62657 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:31:56,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@513a26ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:31:56,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:31:56,794 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:57636, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:31:56,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:31:56,801 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:39248, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:31:56,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] impl.BackupSystemTable(408): write bulk load descriptor to backup test-1690029065625 with 1 entries 2023-07-22T12:31:56,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] impl.BackupSystemTable(1582): writing raw bulk path hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/d1d0c6c0a475cf01e7de961f9001d124/f/acd648b54cf0414a8319c9fc8c54d368_SeqId_4_ for test-1690029065625 d1d0c6c0a475cf01e7de961f9001d124 2023-07-22T12:31:56,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] impl.BackupSystemTable(415): written 1 rows for bulk load of test-1690029065625 2023-07-22T12:31:56,819 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] client.ConnectionImplementation(2095): Closing master protocol: MasterService 2023-07-22T12:31:56,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x6eecc919 to 127.0.0.1:62657 2023-07-22T12:31:56,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:31:56,820 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/staging/jenkins__test-1690029065625__edaj3f48mta1udc6nk152sb3222a80976hc9ve7tnbg92va6ocdtchvs74n584qo/f/79a99e90c2c84e3cb4186b5fa29d3371 into d1d0c6c0a475cf01e7de961f9001d124/f as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/d1d0c6c0a475cf01e7de961f9001d124/f/acd648b54cf0414a8319c9fc8c54d368_SeqId_4_ - updating store file list. 2023-07-22T12:31:56,825 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/d1d0c6c0a475cf01e7de961f9001d124/f/acd648b54cf0414a8319c9fc8c54d368_SeqId_4_ into d1d0c6c0a475cf01e7de961f9001d124/f 2023-07-22T12:31:56,825 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/staging/jenkins__test-1690029065625__edaj3f48mta1udc6nk152sb3222a80976hc9ve7tnbg92va6ocdtchvs74n584qo/f/79a99e90c2c84e3cb4186b5fa29d3371 into d1d0c6c0a475cf01e7de961f9001d124/f (new location: hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/d1d0c6c0a475cf01e7de961f9001d124/f/acd648b54cf0414a8319c9fc8c54d368_SeqId_4_) 2023-07-22T12:31:56,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/staging/jenkins__test-1690029065625__edaj3f48mta1udc6nk152sb3222a80976hc9ve7tnbg92va6ocdtchvs74n584qo/f/79a99e90c2c84e3cb4186b5fa29d3371 2023-07-22T12:31:56,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5d0a5a94 to 127.0.0.1:62657 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:31:56,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@541cef2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:31:56,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:31:56,833 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:57644, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-07-22T12:31:56,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:31:56,838 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:39254, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-07-22T12:31:56,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] impl.BackupSystemTable(388): write bulk load descriptor to backup test-1690029065625 with 1 entries 2023-07-22T12:31:56,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] impl.BackupSystemTable(1503): writing done bulk path hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/d1d0c6c0a475cf01e7de961f9001d124/f/acd648b54cf0414a8319c9fc8c54d368_SeqId_4_ for test-1690029065625 d1d0c6c0a475cf01e7de961f9001d124 2023-07-22T12:31:56,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] impl.BackupSystemTable(394): written 1 rows for bulk load of test-1690029065625 2023-07-22T12:31:56,854 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] client.ConnectionImplementation(2095): Closing master protocol: MasterService 2023-07-22T12:31:56,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5d0a5a94 to 127.0.0.1:62657 2023-07-22T12:31:56,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:31:56,856 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] client.ConnectionImplementation(2095): Closing master protocol: MasterService 2023-07-22T12:31:56,856 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x2a38abef to 127.0.0.1:62657 2023-07-22T12:31:56,856 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:31:56,857 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-07-22T12:31:56,857 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-07-22T12:31:56,857 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(233): Backup: backup_1690029102537 hdfs://localhost:34745/backupUT/backup_1690029102537/default/test-1690029065625/ 2023-07-22T12:31:56,857 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-07-22T12:31:56,876 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreSingleOverwrite Thread=600 (was 598) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:62657@0x2a38abef-SendThread(127.0.0.1:62657) sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:345) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1223) Potentially hanging thread: PacketResponder: BP-821173014-172.31.12.81-1690029049712:blk_1073741891_1067, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: HFileArchiver-3 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:62657@0x5d0a5a94-SendThread(127.0.0.1:62657) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1261) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_842990668_16 at /127.0.0.1:58526 [Receiving block BP-821173014-172.31.12.81-1690029049712:blk_1073741891_1067] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:62657@0x6eecc919-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:62657@0x5d0a5a94-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:62657@0x2a38abef java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1529) org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1512) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1481) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1415) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$493/1448447415.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x69245e7b-shared-pool-9 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x4c100189-shared-pool-24 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:62657@0x6eecc919 java.lang.Object.wait(Native Method) java.lang.Thread.join(Thread.java:1257) java.lang.Thread.join(Thread.java:1331) org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1455) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1485) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1415) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$493/1448447415.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:62657@0x2a38abef-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) Potentially hanging thread: hconnection-0x4c100189-shared-pool-23 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-821173014-172.31.12.81-1690029049712:blk_1073741890_1066, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:62657@0x5d0a5a94 java.lang.Object.wait(Native Method) java.lang.Thread.join(Thread.java:1257) java.lang.Thread.join(Thread.java:1331) org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1455) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1485) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1415) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$493/1448447415.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_842990668_16 at /127.0.0.1:58522 [Receiving block BP-821173014-172.31.12.81-1690029049712:blk_1073741890_1066] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:62657@0x6eecc919-SendThread(127.0.0.1:62657) java.lang.Thread.sleep(Native Method) org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:228) org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1338) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1261) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1319962190_16 at /127.0.0.1:58626 [Waiting for operation #20] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=686 (was 672) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=161 (was 150) - SystemLoadAverage LEAK? -, ProcessCount=169 (was 169), AvailableMemoryMB=16590 (was 16906) 2023-07-22T12:31:56,876 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(130): Thread=600 is superior to 500 2023-07-22T12:31:56,891 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreSingleDNECommand Thread=600, OpenFileDescriptor=686, MaxFileDescriptor=60000, SystemLoadAverage=161, ProcessCount=169, AvailableMemoryMB=16590 2023-07-22T12:31:56,891 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(130): Thread=600 is superior to 500 2023-07-22T12:31:56,891 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] backup.TestFullRestore(280): test restore fails on a single table that does not exist: command-line 2023-07-22T12:31:56,891 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1aa5ea15 to 127.0.0.1:62657 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:31:56,897 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18c7da69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:31:56,902 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:31:56,903 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:39264, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:31:56,904 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:31:56,905 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:57658, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:31:56,916 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-07-22T12:31:56,921 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:31:56,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029116921 (current time:1690029116921). 2023-07-22T12:31:56,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:31:56,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-07-22T12:31:56,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x37461fe3 to 127.0.0.1:62657 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:31:56,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ecb0461, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:31:56,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:31:56,928 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:39280, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:31:56,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x37461fe3 to 127.0.0.1:62657 2023-07-22T12:31:56,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:31:56,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:31:56,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=60, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:31:56,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 60 2023-07-22T12:31:56,932 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:31:56,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=60 2023-07-22T12:31:56,932 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:31:56,935 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:31:57,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=60 2023-07-22T12:31:57,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=60 2023-07-22T12:31:57,341 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:31:57,342 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33}] 2023-07-22T12:31:57,345 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=61, ppid=60, state=RUNNABLE; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33 2023-07-22T12:31:57,443 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-07-22T12:31:57,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43639 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=61 2023-07-22T12:31:57,497 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. 2023-07-22T12:31:57,497 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(2819): Flushing 697cc5ecf0962f5ab9f4126febf7ce33 2/2 column families, dataSize=1.14 KB heapSize=2.22 KB 2023-07-22T12:31:57,501 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/meta/5ea0158e2d574f758c68ee9c20c21191 is 169, key is trslm:hdfs://localhost:34745/backupUT\x00test-1690029065625/meta:log-roll-map/1690029111304/Put/seqid=0 2023-07-22T12:31:57,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=60 2023-07-22T12:31:57,905 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=451 B at sequenceid=28 (bloomFilter=true), to=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/meta/5ea0158e2d574f758c68ee9c20c21191 2023-07-22T12:31:57,912 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 5ea0158e2d574f758c68ee9c20c21191 2023-07-22T12:31:57,915 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/session/62b5d2f1465b492ab802e26f5bf32f6b is 452, key is session:backup_1690029102537/session:context/1690029111729/Put/seqid=0 2023-07-22T12:31:58,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=60 2023-07-22T12:31:58,319 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=721 B at sequenceid=28 (bloomFilter=true), to=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/session/62b5d2f1465b492ab802e26f5bf32f6b 2023-07-22T12:31:58,326 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 62b5d2f1465b492ab802e26f5bf32f6b 2023-07-22T12:31:58,327 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/meta/5ea0158e2d574f758c68ee9c20c21191 as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/5ea0158e2d574f758c68ee9c20c21191 2023-07-22T12:31:58,333 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/5ea0158e2d574f758c68ee9c20c21191, entries=4, sequenceid=28, filesize=5.5 K 2023-07-22T12:31:58,334 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/session/62b5d2f1465b492ab802e26f5bf32f6b as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/62b5d2f1465b492ab802e26f5bf32f6b 2023-07-22T12:31:58,341 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/62b5d2f1465b492ab802e26f5bf32f6b, entries=2, sequenceid=28, filesize=5.4 K 2023-07-22T12:31:58,342 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1172, heapSize ~2.19 KB/2240, currentSize=0 B/0 for 697cc5ecf0962f5ab9f4126febf7ce33 in 845ms, sequenceid=28, compaction requested=true 2023-07-22T12:31:58,342 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(2520): Flush status journal for 697cc5ecf0962f5ab9f4126febf7ce33: 2023-07-22T12:31:58,342 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. for snapshot_backup_system completed. 2023-07-22T12:31:58,342 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33.' region-info for snapshot=snapshot_backup_system 2023-07-22T12:31:58,342 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:31:58,342 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/b580ac8bece04b07a21605e555bdc696, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/5ea0158e2d574f758c68ee9c20c21191] hfiles 2023-07-22T12:31:58,343 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/b580ac8bece04b07a21605e555bdc696 for snapshot=snapshot_backup_system 2023-07-22T12:31:58,343 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/5ea0158e2d574f758c68ee9c20c21191 for snapshot=snapshot_backup_system 2023-07-22T12:31:58,343 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/956010b3eaa64b4fa82ddea4557fac2c, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/0704967b71ce4a5cb0af553e6e47addb, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/62b5d2f1465b492ab802e26f5bf32f6b] hfiles 2023-07-22T12:31:58,343 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/956010b3eaa64b4fa82ddea4557fac2c for snapshot=snapshot_backup_system 2023-07-22T12:31:58,343 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/0704967b71ce4a5cb0af553e6e47addb for snapshot=snapshot_backup_system 2023-07-22T12:31:58,343 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/62b5d2f1465b492ab802e26f5bf32f6b for snapshot=snapshot_backup_system 2023-07-22T12:31:58,751 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. 2023-07-22T12:31:58,751 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=61 2023-07-22T12:31:58,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster(3994): Remote procedure done, pid=61 2023-07-22T12:31:58,752 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 697cc5ecf0962f5ab9f4126febf7ce33 2023-07-22T12:31:58,753 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=61, ppid=60, state=RUNNABLE; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33 2023-07-22T12:31:58,756 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=61, resume processing ppid=60 2023-07-22T12:31:58,756 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=61, ppid=60, state=SUCCESS; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33 in 1.4120 sec 2023-07-22T12:31:58,756 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:31:58,757 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:31:58,757 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:31:58,757 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-07-22T12:31:58,758 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_backup_system 2023-07-22T12:31:59,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=60 2023-07-22T12:31:59,172 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:31:59,180 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:31:59,181 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_backup_system 2023-07-22T12:32:00,008 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:32:00,008 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 60 2023-07-22T12:32:00,009 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=60, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 3.0790 sec 2023-07-22T12:32:00,102 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 595143905c5328f814fba73ad7017c76 changed from -1.0 to 0.0, refreshing cache 2023-07-22T12:32:01,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=60 2023-07-22T12:32:01,038 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: backup:system, procId: 60 completed 2023-07-22T12:32:01,038 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(115): Backup backup_1690029116897 started at 1690029121038. 2023-07-22T12:32:01,040 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(119): Backup session backup_1690029116897 has been started. 2023-07-22T12:32:01,045 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-07-22T12:32:01,046 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-07-22T12:32:01,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-07-22T12:32:01,046 INFO [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-07-22T12:32:01,046 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-07-22T12:32:01,047 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-07-22T12:32:01,047 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-07-22T12:32:01,048 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-07-22T12:32:01,048 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:32:01,049 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): regionserver:43639-0x100005e31340001, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-07-22T12:32:01,050 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:01,050 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:32:01,050 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:32:01,050 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:01,050 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-07-22T12:32:01,050 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-07-22T12:32:01,050 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43639-0x100005e31340001, quorum=127.0.0.1:62657, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-07-22T12:32:01,051 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-07-22T12:32:01,051 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:32:01,051 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-07-22T12:32:01,051 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-07-22T12:32:01,051 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-07-22T12:32:01,051 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-07-22T12:32:01,051 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-07-22T12:32:01,052 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-07-22T12:32:01,052 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-07-22T12:32:01,052 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,43639,1690029056327' joining acquired barrier for procedure (rolllog) in zk 2023-07-22T12:32:01,054 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:01,054 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-07-22T12:32:01,054 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:01,054 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:32:01,054 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:32:01,054 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43639-0x100005e31340001, quorum=127.0.0.1:62657, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-07-22T12:32:01,054 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-07-22T12:32:01,055 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:32:01,055 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:32:01,055 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:32:01,055 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:01,056 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:32:01,056 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,43639,1690029056327' joining acquired barrier for procedure 'rolllog' on coordinator 2023-07-22T12:32:01,056 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@26738989[Count = 0] remaining members to acquire global barrier 2023-07-22T12:32:01,056 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-07-22T12:32:01,056 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-07-22T12:32:01,057 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): regionserver:43639-0x100005e31340001, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-07-22T12:32:01,057 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-07-22T12:32:01,057 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-07-22T12:32:01,057 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-07-22T12:32:01,057 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-07-22T12:32:01,057 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:01,057 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-07-22T12:32:01,057 DEBUG [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:01,057 INFO [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1690029106751 highest: 1690029106751 on jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:01,058 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta:.meta(num 1690029106733) roll requested 2023-07-22T12:32:01,069 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029121058.meta, exclude list is [], retry=0 2023-07-22T12:32:01,073 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK] 2023-07-22T12:32:01,075 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029106733.meta with entries=12, filesize=3.07 KB; new WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029121058.meta 2023-07-22T12:32:01,075 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK]] 2023-07-22T12:32:01,075 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029106733.meta is not closed yet, will try archiving it next time 2023-07-22T12:32:01,075 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C43639%2C1690029056327:(num 1690029106751) roll requested 2023-07-22T12:32:01,077 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029106733.meta not finished, retry = 0 2023-07-22T12:32:01,087 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029121076, exclude list is [], retry=0 2023-07-22T12:32:01,090 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK] 2023-07-22T12:32:01,091 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029106751 with entries=17, filesize=5.25 KB; new WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029121076 2023-07-22T12:32:01,091 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK]] 2023-07-22T12:32:01,091 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029106751 is not closed yet, will try archiving it next time 2023-07-22T12:32:01,098 DEBUG [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-07-22T12:32:01,098 INFO [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1690029121076 on jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:01,107 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-07-22T12:32:01,107 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,43639,1690029056327' in zk 2023-07-22T12:32:01,109 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:01,109 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-07-22T12:32:01,109 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:32:01,109 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-07-22T12:32:01,110 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:01,110 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:32:01,110 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:32:01,110 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:32:01,111 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:32:01,111 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:32:01,111 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:01,111 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:32:01,112 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:32:01,112 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:01,112 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,43639,1690029056327': 2023-07-22T12:32:01,112 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,43639,1690029056327' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-07-22T12:32:01,112 INFO [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-07-22T12:32:01,113 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-07-22T12:32:01,113 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-07-22T12:32:01,113 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-07-22T12:32:01,113 INFO [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-07-22T12:32:01,115 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): regionserver:43639-0x100005e31340001, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-07-22T12:32:01,115 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-07-22T12:32:01,115 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): regionserver:43639-0x100005e31340001, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-07-22T12:32:01,115 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-07-22T12:32:01,115 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-07-22T12:32:01,115 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:32:01,115 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:32:01,115 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:32:01,116 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:32:01,116 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:32:01,116 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:32:01,116 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:01,116 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:32:01,116 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:32:01,116 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:32:01,117 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:32:01,117 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:01,117 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:01,117 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:32:01,118 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:32:01,118 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:01,121 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:01,121 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): regionserver:43639-0x100005e31340001, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-07-22T12:32:01,121 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-07-22T12:32:01,121 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-07-22T12:32:01,121 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): regionserver:43639-0x100005e31340001, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-07-22T12:32:01,121 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:32:01,121 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:01,121 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:32:01,121 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-07-22T12:32:01,121 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:32:01,121 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-07-22T12:32:01,121 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-07-22T12:32:01,122 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-07-22T12:32:01,122 DEBUG [Listener at localhost/39427-EventThread {}] zookeeper.ZKWatcher(604): master:37923-0x100005e31340000, quorum=127.0.0.1:62657, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-07-22T12:32:01,122 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:32:01,122 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:32:01,122 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] client.HBaseAdmin(2751): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-07-22T12:32:01,123 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] client.HBaseAdmin(2759): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-07-22T12:32:01,223 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] client.HBaseAdmin(2765): Getting current status of procedure from master... 2023-07-22T12:32:01,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-07-22T12:32:01,227 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1690029121226_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } 2023-07-22T12:32:01,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029121227 (current time:1690029121227). 2023-07-22T12:32:01,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:32:01,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1690029121226_default_test-1690029065625 VERSION not specified, setting to 2 2023-07-22T12:32:01,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0c9a5565 to 127.0.0.1:62657 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:32:01,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ba4ea12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:32:01,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:32:01,235 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:44046, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:32:01,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0c9a5565 to 127.0.0.1:62657 2023-07-22T12:32:01,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:32:01,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:32:01,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=62, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1690029121226_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } 2023-07-22T12:32:01,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1690029121226_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 }, snapshot procedure id = 62 2023-07-22T12:32:01,240 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1690029121226_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:32:01,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=62 2023-07-22T12:32:01,241 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1690029121226_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:32:01,244 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1690029121226_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:32:01,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=62 2023-07-22T12:32:01,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=62 2023-07-22T12:32:01,653 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1690029121226_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:32:01,653 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE; SnapshotRegionProcedure d1d0c6c0a475cf01e7de961f9001d124}] 2023-07-22T12:32:01,655 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=63, ppid=62, state=RUNNABLE; SnapshotRegionProcedure d1d0c6c0a475cf01e7de961f9001d124 2023-07-22T12:32:01,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43639 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=63 2023-07-22T12:32:01,807 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1690029065625,,1690029112094.d1d0c6c0a475cf01e7de961f9001d124. 2023-07-22T12:32:01,807 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.HRegion(2520): Flush status journal for d1d0c6c0a475cf01e7de961f9001d124: 2023-07-22T12:32:01,807 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1690029065625,,1690029112094.d1d0c6c0a475cf01e7de961f9001d124. for snapshot_1690029121226_default_test-1690029065625 completed. 2023-07-22T12:32:01,807 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(240): Storing 'test-1690029065625,,1690029112094.d1d0c6c0a475cf01e7de961f9001d124.' region-info for snapshot=snapshot_1690029121226_default_test-1690029065625 2023-07-22T12:32:01,807 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:32:01,808 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/d1d0c6c0a475cf01e7de961f9001d124/f/acd648b54cf0414a8319c9fc8c54d368_SeqId_4_] hfiles 2023-07-22T12:32:01,808 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/d1d0c6c0a475cf01e7de961f9001d124/f/acd648b54cf0414a8319c9fc8c54d368_SeqId_4_ for snapshot=snapshot_1690029121226_default_test-1690029065625 2023-07-22T12:32:01,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=62 2023-07-22T12:32:02,215 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1690029065625,,1690029112094.d1d0c6c0a475cf01e7de961f9001d124. 2023-07-22T12:32:02,215 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=63 2023-07-22T12:32:02,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster(3994): Remote procedure done, pid=63 2023-07-22T12:32:02,215 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1690029121226_default_test-1690029065625 on region d1d0c6c0a475cf01e7de961f9001d124 2023-07-22T12:32:02,217 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=63, ppid=62, state=RUNNABLE; SnapshotRegionProcedure d1d0c6c0a475cf01e7de961f9001d124 2023-07-22T12:32:02,219 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=63, resume processing ppid=62 2023-07-22T12:32:02,219 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1690029121226_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:32:02,219 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=63, ppid=62, state=SUCCESS; SnapshotRegionProcedure d1d0c6c0a475cf01e7de961f9001d124 in 564 msec 2023-07-22T12:32:02,220 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1690029121226_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:32:02,220 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1690029121226_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:32:02,220 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1690029121226_default_test-1690029065625 2023-07-22T12:32:02,221 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_1690029121226_default_test-1690029065625 2023-07-22T12:32:02,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=62 2023-07-22T12:32:02,630 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1690029121226_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:32:02,636 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1690029121226_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:32:02,636 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_1690029121226_default_test-1690029065625 to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029121226_default_test-1690029065625 2023-07-22T12:32:03,225 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1690029065625 2023-07-22T12:32:03,225 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test-1690029065625 Metrics about Tables on a single HBase RegionServer 2023-07-22T12:32:03,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=62 2023-07-22T12:32:03,454 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1690029121226_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:32:03,454 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1690029121226_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 }, snapshot procedure id = 62 2023-07-22T12:32:03,455 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=62, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1690029121226_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } in 2.2160 sec 2023-07-22T12:32:05,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=62 2023-07-22T12:32:05,348 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: default:test-1690029065625, procId: 62 completed 2023-07-22T12:32:05,348 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1690029116897 2023-07-22T12:32:05,348 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-07-22T12:32:05,348 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-07-22T12:32:05,348 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1690029116897_test-1690029065625 2023-07-22T12:32:05,348 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1690029121226_default_test-1690029065625 to hdfs://localhost:34745/backupUT/backup_1690029116897/default/test-1690029065625/ 2023-07-22T12:32:05,348 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-07-22T12:32:05,373 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:34745, inputRoot=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947 2023-07-22T12:32:05,373 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,37923,1690029053050_1993384699_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:34745/backupUT/backup_1690029116897/default/test-1690029065625, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:34745/backupUT/backup_1690029116897/default/test-1690029065625/.hbase-snapshot/.tmp/snapshot_1690029121226_default_test-1690029065625 2023-07-22T12:32:05,373 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:34745, inputRoot=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947, snapshotDir=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029121226_default_test-1690029065625. 2023-07-22T12:32:05,381 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029121226_default_test-1690029065625 to hdfs://localhost:34745/backupUT/backup_1690029116897/default/test-1690029065625/.hbase-snapshot/.tmp/snapshot_1690029121226_default_test-1690029065625 2023-07-22T12:32:05,795 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:05,795 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:05,796 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:05,796 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:06,759 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/hadoop-6130917491547367944.jar 2023-07-22T12:32:06,759 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:06,760 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:06,760 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:06,761 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:06,761 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:06,762 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:06,762 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:06,763 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:06,763 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:06,764 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:32:06,764 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-07-22T12:32:06,765 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-07-22T12:32:06,765 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-07-22T12:32:06,766 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-07-22T12:32:06,766 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-07-22T12:32:06,766 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-07-22T12:32:06,767 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-07-22T12:32:06,767 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-07-22T12:32:06,768 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-07-22T12:32:06,769 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-07-22T12:32:06,769 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-07-22T12:32:06,769 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:32:06,770 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:32:06,771 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:32:06,771 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:32:06,771 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:32:06,772 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:32:06,772 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:32:06,799 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-07-22T12:32:06,818 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1690029121226_default_test-1690029065625' hfile list 2023-07-22T12:32:06,821 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-07-22T12:32:06,893 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029126859/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-07-22T12:32:06,893 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029126859/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-07-22T12:32:06,979 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-07-22T12:32:07,007 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/d1d0c6c0a475cf01e7de961f9001d124/f/acd648b54cf0414a8319c9fc8c54d368_SeqId_4_ output=hdfs://localhost:34745/backupUT/backup_1690029116897/default/test-1690029065625/archive/data/default/test-1690029065625/d1d0c6c0a475cf01e7de961f9001d124/f/acd648b54cf0414a8319c9fc8c54d368_SeqId_4_ 2023-07-22T12:32:07,007 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8724 (8.5 K) time=0sec 8.320M/sec 2023-07-22T12:32:07,954 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-07-22T12:32:07,956 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-07-22T12:32:07,962 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1690029121226_default_test-1690029065625 2023-07-22T12:32:07,962 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1690029121226_default_test-1690029065625 finished. 2023-07-22T12:32:07,963 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(1014): test-1690029065625 2023-07-22T12:32:07,969 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029116897 2023-07-22T12:32:07,969 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-07-22T12:32:08,375 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:34745/backupUT/backup_1690029116897/.backup.manifest 2023-07-22T12:32:08,375 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(358): Backup backup_1690029116897 finished: type=FULL,tablelist=test-1690029065625,targetRootDir=hdfs://localhost:34745/backupUT,startts=1690029121038,completets=1690029127969,bytescopied=0 2023-07-22T12:32:08,375 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-07-22T12:32:08,375 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1690029121226_default_test-1690029065625 2023-07-22T12:32:08,376 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1690029121226_default_test-1690029065625" 2023-07-22T12:32:08,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1690029121226_default_test-1690029065625 2023-07-22T12:32:08,380 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1690029121226_default_test-1690029065625 for backup backup_1690029116897 succeeded. 2023-07-22T12:32:08,381 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-07-22T12:32:08,385 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-07-22T12:32:08,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-07-22T12:32:08,387 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-07-22T12:32:08,389 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-07-22T12:32:08,394 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(376): Backup backup_1690029116897 completed. 2023-07-22T12:32:08,395 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] client.ConnectionImplementation(2095): Closing master protocol: MasterService 2023-07-22T12:32:08,395 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1aa5ea15 to 127.0.0.1:62657 2023-07-22T12:32:08,395 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:32:08,405 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] backup.TestFullRestore(285): backup complete You have to specify either set name or table list to restore Usage: hbase restore [options] backup_path Path to a backup destination root backup_id Backup image ID to restore table(s) Comma-separated list of tables to restore Options: -c Check restore sequence and dependencies only (does not execute the command) -d Enable debug loggings -h,--help Show usage -m A comma separated list of target tables. If specified, each table in must have a mapping -o Overwrite data if any of the restore target tables exists -q Yarn queue name to run backup restore command on -s Backup set to restore, mutually exclusive with -t (table list) -t Table name list, comma-separated. To enable backup, in hbase-site.xml, set: hbase.backup.enable=true hbase.master.logcleaner.plugins=YOUR_PLUGINS,org.apache.hadoop.hbase.backup.master.BackupLogCleaner hbase.procedure.master.classes=YOUR_CLASSES,org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager hbase.procedure.regionserver.classes=YOUR_CLASSES,org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager hbase.coprocessor.region.classes=YOUR_CLASSES,org.apache.hadoop.hbase.backup.BackupObserver and restart the cluster For more information please see http://hbase.apache.org/book.html#backuprestore 2023-07-22T12:32:08,435 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreSingleDNECommand Thread=591 (was 600), OpenFileDescriptor=674 (was 686), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=165 (was 161) - SystemLoadAverage LEAK? -, ProcessCount=169 (was 169), AvailableMemoryMB=16478 (was 16590) 2023-07-22T12:32:08,435 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(130): Thread=591 is superior to 500 2023-07-22T12:32:08,450 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreCheckCommand Thread=591, OpenFileDescriptor=674, MaxFileDescriptor=60000, SystemLoadAverage=165, ProcessCount=169, AvailableMemoryMB=16477 2023-07-22T12:32:08,450 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(130): Thread=591 is superior to 500 2023-07-22T12:32:08,450 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] backup.TestFullRestore(97): test full restore on a single table: command-line, check only 2023-07-22T12:32:08,459 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25338080, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:32:08,463 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:32:08,465 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:44048, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:32:08,466 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:32:08,468 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:52770, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:32:08,479 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-07-22T12:32:08,485 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:32:08,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029128485 (current time:1690029128485). 2023-07-22T12:32:08,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:32:08,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-07-22T12:32:08,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@539b8960, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:32:08,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:32:08,493 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:44054, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:32:08,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:32:08,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:32:08,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=64, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:32:08,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 64 2023-07-22T12:32:08,498 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:32:08,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=64 2023-07-22T12:32:08,498 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:32:08,500 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:32:08,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=64 2023-07-22T12:32:08,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=64 2023-07-22T12:32:08,909 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:32:08,909 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33}] 2023-07-22T12:32:08,911 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33 2023-07-22T12:32:09,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43639 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=65 2023-07-22T12:32:09,063 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. 2023-07-22T12:32:09,063 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(2819): Flushing 697cc5ecf0962f5ab9f4126febf7ce33 2/2 column families, dataSize=1.14 KB heapSize=2.22 KB 2023-07-22T12:32:09,067 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/meta/a6d2695c5574440ea495f58a7c66ed03 is 169, key is trslm:hdfs://localhost:34745/backupUT\x00test-1690029065625/meta:log-roll-map/1690029127965/Put/seqid=0 2023-07-22T12:32:09,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=64 2023-07-22T12:32:09,471 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=451 B at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/meta/a6d2695c5574440ea495f58a7c66ed03 2023-07-22T12:32:09,477 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a6d2695c5574440ea495f58a7c66ed03 2023-07-22T12:32:09,480 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/session/c2fdeacbf4184719a11904adfeaf1fd8 is 452, key is session:backup_1690029116897/session:context/1690029128388/Put/seqid=0 2023-07-22T12:32:09,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=64 2023-07-22T12:32:09,885 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=721 B at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/session/c2fdeacbf4184719a11904adfeaf1fd8 2023-07-22T12:32:09,891 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: c2fdeacbf4184719a11904adfeaf1fd8 2023-07-22T12:32:09,892 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/meta/a6d2695c5574440ea495f58a7c66ed03 as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/a6d2695c5574440ea495f58a7c66ed03 2023-07-22T12:32:09,897 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/a6d2695c5574440ea495f58a7c66ed03, entries=4, sequenceid=39, filesize=5.5 K 2023-07-22T12:32:09,898 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/session/c2fdeacbf4184719a11904adfeaf1fd8 as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/c2fdeacbf4184719a11904adfeaf1fd8 2023-07-22T12:32:09,902 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/c2fdeacbf4184719a11904adfeaf1fd8, entries=2, sequenceid=39, filesize=5.4 K 2023-07-22T12:32:09,903 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1172, heapSize ~2.19 KB/2240, currentSize=0 B/0 for 697cc5ecf0962f5ab9f4126febf7ce33 in 840ms, sequenceid=39, compaction requested=true 2023-07-22T12:32:09,903 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(2520): Flush status journal for 697cc5ecf0962f5ab9f4126febf7ce33: 2023-07-22T12:32:09,903 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. for snapshot_backup_system completed. 2023-07-22T12:32:09,903 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33.' region-info for snapshot=snapshot_backup_system 2023-07-22T12:32:09,904 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:32:09,904 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/b580ac8bece04b07a21605e555bdc696, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/5ea0158e2d574f758c68ee9c20c21191, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/a6d2695c5574440ea495f58a7c66ed03] hfiles 2023-07-22T12:32:09,904 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/b580ac8bece04b07a21605e555bdc696 for snapshot=snapshot_backup_system 2023-07-22T12:32:09,904 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/5ea0158e2d574f758c68ee9c20c21191 for snapshot=snapshot_backup_system 2023-07-22T12:32:09,904 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/a6d2695c5574440ea495f58a7c66ed03 for snapshot=snapshot_backup_system 2023-07-22T12:32:09,904 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/956010b3eaa64b4fa82ddea4557fac2c, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/0704967b71ce4a5cb0af553e6e47addb, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/62b5d2f1465b492ab802e26f5bf32f6b, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/c2fdeacbf4184719a11904adfeaf1fd8] hfiles 2023-07-22T12:32:09,904 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (1/4): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/956010b3eaa64b4fa82ddea4557fac2c for snapshot=snapshot_backup_system 2023-07-22T12:32:09,904 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (2/4): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/0704967b71ce4a5cb0af553e6e47addb for snapshot=snapshot_backup_system 2023-07-22T12:32:09,904 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (3/4): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/62b5d2f1465b492ab802e26f5bf32f6b for snapshot=snapshot_backup_system 2023-07-22T12:32:09,904 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (4/4): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/c2fdeacbf4184719a11904adfeaf1fd8 for snapshot=snapshot_backup_system 2023-07-22T12:32:10,309 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. 2023-07-22T12:32:10,309 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=65 2023-07-22T12:32:10,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster(3994): Remote procedure done, pid=65 2023-07-22T12:32:10,310 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 697cc5ecf0962f5ab9f4126febf7ce33 2023-07-22T12:32:10,311 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33 2023-07-22T12:32:10,313 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=65, resume processing ppid=64 2023-07-22T12:32:10,313 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:32:10,313 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=65, ppid=64, state=SUCCESS; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33 in 1.4030 sec 2023-07-22T12:32:10,313 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:32:10,314 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:32:10,314 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-07-22T12:32:10,315 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_backup_system 2023-07-22T12:32:10,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=64 2023-07-22T12:32:10,725 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:32:10,733 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:32:10,734 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_backup_system 2023-07-22T12:32:11,551 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:32:11,551 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 64 2023-07-22T12:32:11,552 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=64, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 3.0560 sec 2023-07-22T12:32:12,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=64 2023-07-22T12:32:12,604 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(115): Backup backup_1690029128459 started at 1690029132604. 2023-07-22T12:32:12,606 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(119): Backup session backup_1690029128459 has been started. 2023-07-22T12:32:12,611 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-07-22T12:32:12,611 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-07-22T12:32:12,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-07-22T12:32:12,611 INFO [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-07-22T12:32:12,611 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-07-22T12:32:12,612 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-07-22T12:32:12,612 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-07-22T12:32:12,613 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:32:12,616 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:12,616 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:32:12,616 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:32:12,616 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-07-22T12:32:12,616 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-07-22T12:32:12,617 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-07-22T12:32:12,617 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:32:12,617 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-07-22T12:32:12,617 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-07-22T12:32:12,617 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-07-22T12:32:12,617 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-07-22T12:32:12,617 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-07-22T12:32:12,618 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-07-22T12:32:12,618 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-07-22T12:32:12,618 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,43639,1690029056327' joining acquired barrier for procedure (rolllog) in zk 2023-07-22T12:32:12,620 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-07-22T12:32:12,620 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:12,620 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:32:12,620 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:32:12,620 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-07-22T12:32:12,621 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:32:12,621 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:32:12,621 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:32:12,621 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:12,622 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:32:12,622 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,43639,1690029056327' joining acquired barrier for procedure 'rolllog' on coordinator 2023-07-22T12:32:12,622 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@58f026c5[Count = 0] remaining members to acquire global barrier 2023-07-22T12:32:12,622 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-07-22T12:32:12,622 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-07-22T12:32:12,623 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-07-22T12:32:12,623 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-07-22T12:32:12,623 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-07-22T12:32:12,623 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-07-22T12:32:12,623 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-07-22T12:32:12,623 DEBUG [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:12,623 INFO [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1690029121076 highest: 1690029121076 on jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:12,624 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta:.meta(num 1690029121058) roll requested 2023-07-22T12:32:12,638 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029132624.meta, exclude list is [], retry=0 2023-07-22T12:32:12,641 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK] 2023-07-22T12:32:12,643 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029121058.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029132624.meta 2023-07-22T12:32:12,643 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK]] 2023-07-22T12:32:12,643 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029121058.meta is not closed yet, will try archiving it next time 2023-07-22T12:32:12,643 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C43639%2C1690029056327:(num 1690029121076) roll requested 2023-07-22T12:32:12,646 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029121058.meta to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/oldWALs/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029121058.meta 2023-07-22T12:32:12,655 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029132643, exclude list is [], retry=0 2023-07-22T12:32:12,659 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK] 2023-07-22T12:32:12,660 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029121076 with entries=10, filesize=2.31 KB; new WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029132643 2023-07-22T12:32:12,660 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK]] 2023-07-22T12:32:12,661 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029121076 is not closed yet, will try archiving it next time 2023-07-22T12:32:12,664 DEBUG [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-07-22T12:32:12,664 INFO [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1690029132643 on jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:12,675 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-07-22T12:32:12,675 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,43639,1690029056327' in zk 2023-07-22T12:32:12,677 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-07-22T12:32:12,677 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:12,678 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:32:12,678 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:32:12,678 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-07-22T12:32:12,678 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:32:12,679 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:32:12,679 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:32:12,679 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:32:12,680 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:12,680 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:32:12,680 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:32:12,680 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:12,681 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,43639,1690029056327': 2023-07-22T12:32:12,681 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,43639,1690029056327' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-07-22T12:32:12,681 INFO [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-07-22T12:32:12,681 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-07-22T12:32:12,681 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-07-22T12:32:12,681 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-07-22T12:32:12,681 INFO [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-07-22T12:32:12,685 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-07-22T12:32:12,685 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-07-22T12:32:12,685 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:32:12,685 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:32:12,685 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:32:12,685 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:32:12,685 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:32:12,685 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:32:12,686 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:32:12,686 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:32:12,686 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:32:12,686 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:32:12,687 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:12,687 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:32:12,687 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:32:12,688 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:12,691 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:32:12,691 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:32:12,691 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:32:12,691 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-07-22T12:32:12,691 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-07-22T12:32:12,692 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:32:12,692 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:32:12,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-07-22T12:32:12,796 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1690029132795_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } 2023-07-22T12:32:12,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029132796 (current time:1690029132796). 2023-07-22T12:32:12,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:32:12,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1690029132795_default_test-1690029065625 VERSION not specified, setting to 2 2023-07-22T12:32:12,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d827591, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:32:12,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:32:12,805 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:39810, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:32:12,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:32:12,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:32:12,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=66, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1690029132795_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } 2023-07-22T12:32:12,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1690029132795_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 }, snapshot procedure id = 66 2023-07-22T12:32:12,808 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1690029132795_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:32:12,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=66 2023-07-22T12:32:12,809 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1690029132795_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:32:12,811 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1690029132795_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:32:12,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=66 2023-07-22T12:32:13,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=66 2023-07-22T12:32:13,216 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1690029132795_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:32:13,217 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE; SnapshotRegionProcedure d1d0c6c0a475cf01e7de961f9001d124}] 2023-07-22T12:32:13,219 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=67, ppid=66, state=RUNNABLE; SnapshotRegionProcedure d1d0c6c0a475cf01e7de961f9001d124 2023-07-22T12:32:13,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43639 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=67 2023-07-22T12:32:13,370 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1690029065625,,1690029112094.d1d0c6c0a475cf01e7de961f9001d124. 2023-07-22T12:32:13,370 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.HRegion(2520): Flush status journal for d1d0c6c0a475cf01e7de961f9001d124: 2023-07-22T12:32:13,370 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1690029065625,,1690029112094.d1d0c6c0a475cf01e7de961f9001d124. for snapshot_1690029132795_default_test-1690029065625 completed. 2023-07-22T12:32:13,370 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] snapshot.SnapshotManifest(240): Storing 'test-1690029065625,,1690029112094.d1d0c6c0a475cf01e7de961f9001d124.' region-info for snapshot=snapshot_1690029132795_default_test-1690029065625 2023-07-22T12:32:13,371 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:32:13,371 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/d1d0c6c0a475cf01e7de961f9001d124/f/acd648b54cf0414a8319c9fc8c54d368_SeqId_4_] hfiles 2023-07-22T12:32:13,371 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/d1d0c6c0a475cf01e7de961f9001d124/f/acd648b54cf0414a8319c9fc8c54d368_SeqId_4_ for snapshot=snapshot_1690029132795_default_test-1690029065625 2023-07-22T12:32:13,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=66 2023-07-22T12:32:13,776 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1690029065625,,1690029112094.d1d0c6c0a475cf01e7de961f9001d124. 2023-07-22T12:32:13,777 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=67 2023-07-22T12:32:13,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster(3994): Remote procedure done, pid=67 2023-07-22T12:32:13,777 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1690029132795_default_test-1690029065625 on region d1d0c6c0a475cf01e7de961f9001d124 2023-07-22T12:32:13,778 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=67, ppid=66, state=RUNNABLE; SnapshotRegionProcedure d1d0c6c0a475cf01e7de961f9001d124 2023-07-22T12:32:13,780 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=67, resume processing ppid=66 2023-07-22T12:32:13,780 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=67, ppid=66, state=SUCCESS; SnapshotRegionProcedure d1d0c6c0a475cf01e7de961f9001d124 in 562 msec 2023-07-22T12:32:13,780 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1690029132795_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:32:13,781 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1690029132795_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:32:13,781 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1690029132795_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:32:13,781 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1690029132795_default_test-1690029065625 2023-07-22T12:32:13,782 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_1690029132795_default_test-1690029065625 2023-07-22T12:32:13,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=66 2023-07-22T12:32:14,191 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1690029132795_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:32:14,197 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1690029132795_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:32:14,198 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_1690029132795_default_test-1690029065625 to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029132795_default_test-1690029065625 2023-07-22T12:32:14,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=66 2023-07-22T12:32:15,015 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1690029132795_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:32:15,015 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1690029132795_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 }, snapshot procedure id = 66 2023-07-22T12:32:15,016 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=66, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1690029132795_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } in 2.2090 sec 2023-07-22T12:32:16,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=66 2023-07-22T12:32:16,915 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1690029128459 2023-07-22T12:32:16,915 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-07-22T12:32:16,915 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-07-22T12:32:16,915 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1690029128459_test-1690029065625 2023-07-22T12:32:16,915 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1690029132795_default_test-1690029065625 to hdfs://localhost:34745/backupUT/backup_1690029128459/default/test-1690029065625/ 2023-07-22T12:32:16,915 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-07-22T12:32:16,939 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:34745, inputRoot=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947 2023-07-22T12:32:16,939 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,37923,1690029053050_-1453421323_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:34745/backupUT/backup_1690029128459/default/test-1690029065625, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:34745/backupUT/backup_1690029128459/default/test-1690029065625/.hbase-snapshot/.tmp/snapshot_1690029132795_default_test-1690029065625 2023-07-22T12:32:16,939 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:34745, inputRoot=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947, snapshotDir=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029132795_default_test-1690029065625. 2023-07-22T12:32:16,947 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029132795_default_test-1690029065625 to hdfs://localhost:34745/backupUT/backup_1690029128459/default/test-1690029065625/.hbase-snapshot/.tmp/snapshot_1690029132795_default_test-1690029065625 2023-07-22T12:32:17,370 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:17,370 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:17,371 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:17,371 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:18,393 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/hadoop-2062094701249600814.jar 2023-07-22T12:32:18,393 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:18,394 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:18,394 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:18,395 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:18,395 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:18,395 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:18,396 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:18,396 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:18,396 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:18,397 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:32:18,398 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-07-22T12:32:18,398 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-07-22T12:32:18,398 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-07-22T12:32:18,399 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-07-22T12:32:18,399 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-07-22T12:32:18,400 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-07-22T12:32:18,400 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-07-22T12:32:18,401 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-07-22T12:32:18,401 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-07-22T12:32:18,402 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-07-22T12:32:18,402 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-07-22T12:32:18,403 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:32:18,404 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:32:18,405 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:32:18,405 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:32:18,406 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:32:18,407 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:32:18,408 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:32:18,467 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1690029132795_default_test-1690029065625' hfile list 2023-07-22T12:32:18,470 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-07-22T12:32:18,562 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029138521/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-07-22T12:32:18,562 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029138521/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-07-22T12:32:18,621 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-07-22T12:32:18,646 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/d1d0c6c0a475cf01e7de961f9001d124/f/acd648b54cf0414a8319c9fc8c54d368_SeqId_4_ output=hdfs://localhost:34745/backupUT/backup_1690029128459/default/test-1690029065625/archive/data/default/test-1690029065625/d1d0c6c0a475cf01e7de961f9001d124/f/acd648b54cf0414a8319c9fc8c54d368_SeqId_4_ 2023-07-22T12:32:18,646 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8724 (8.5 K) time=0sec 8.320M/sec 2023-07-22T12:32:19,597 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-07-22T12:32:19,599 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-07-22T12:32:19,605 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1690029132795_default_test-1690029065625 2023-07-22T12:32:19,605 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1690029132795_default_test-1690029065625 finished. 2023-07-22T12:32:19,605 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(1014): test-1690029065625 2023-07-22T12:32:19,611 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029128459 2023-07-22T12:32:19,611 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-07-22T12:32:20,016 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:34745/backupUT/backup_1690029128459/.backup.manifest 2023-07-22T12:32:20,016 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(358): Backup backup_1690029128459 finished: type=FULL,tablelist=test-1690029065625,targetRootDir=hdfs://localhost:34745/backupUT,startts=1690029132604,completets=1690029139611,bytescopied=0 2023-07-22T12:32:20,016 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-07-22T12:32:20,016 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1690029132795_default_test-1690029065625 2023-07-22T12:32:20,017 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1690029132795_default_test-1690029065625" 2023-07-22T12:32:20,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1690029132795_default_test-1690029065625 2023-07-22T12:32:20,021 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1690029132795_default_test-1690029065625 for backup backup_1690029128459 succeeded. 2023-07-22T12:32:20,021 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-07-22T12:32:20,026 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-07-22T12:32:20,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-07-22T12:32:20,028 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-07-22T12:32:20,030 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-07-22T12:32:20,036 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(376): Backup backup_1690029128459 completed. 2023-07-22T12:32:20,037 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:32:20,037 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] backup.TestFullRestore(101): backup complete 2023-07-22T12:32:20,049 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] backup.RestoreDriver(108): Found -check option in restore command, will check and verify the dependencies 2023-07-22T12:32:20,060 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66795974, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:32:20,061 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34745/backupUT/backup_1690029128459/.backup.manifest 2023-07-22T12:32:20,064 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029128459/.backup.manifest 2023-07-22T12:32:20,064 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] util.BackupUtils(677): Dependent image(s) from old to new: 2023-07-22T12:32:20,064 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] util.BackupUtils(686): Backup image: backup_1690029128459 for 'test-1690029065625' is available 2023-07-22T12:32:20,064 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupAdminImpl(511): Checking backup images: OK 2023-07-22T12:32:20,064 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:32:20,082 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreCheckCommand Thread=595 (was 591) Potentially hanging thread: PacketResponder: BP-821173014-172.31.12.81-1690029049712:blk_1073741932_1108, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:62657@0x32bff0cc-SendThread(127.0.0.1:62657) java.lang.Thread.sleep(Native Method) org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:228) org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1338) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1261) Potentially hanging thread: hconnection-0x69245e7b-shared-pool-13 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_842990668_16 at /127.0.0.1:44416 [Receiving block BP-821173014-172.31.12.81-1690029049712:blk_1073741931_1107] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:62657@0x2fd65f83-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) Potentially hanging thread: hconnection-0x4c100189-shared-pool-30 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_458691661_16 at /127.0.0.1:44460 [Waiting for operation #8] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x69245e7b-shared-pool-12 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:62657@0x2fd65f83-SendThread(127.0.0.1:62657) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1261) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:62657@0x32bff0cc-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:62657@0x32bff0cc java.lang.Object.wait(Native Method) java.lang.Thread.join(Thread.java:1257) java.lang.Thread.join(Thread.java:1331) org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1455) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1485) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1415) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$493/1448447415.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_842990668_16 at /127.0.0.1:44420 [Receiving block BP-821173014-172.31.12.81-1690029049712:blk_1073741932_1108] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:62657@0x2fd65f83 java.lang.Object.wait(Native Method) java.lang.Thread.join(Thread.java:1257) java.lang.Thread.join(Thread.java:1331) org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1455) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1485) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1415) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$493/1448447415.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: WAL-Archive-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1319962190_16 at /127.0.0.1:44380 [Waiting for operation #22] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-821173014-172.31.12.81-1690029049712:blk_1073741931_1107, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=677 (was 674) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=160 (was 165), ProcessCount=169 (was 169), AvailableMemoryMB=16431 (was 16477) 2023-07-22T12:32:20,082 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(130): Thread=595 is superior to 500 2023-07-22T12:32:20,097 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreSingle Thread=595, OpenFileDescriptor=677, MaxFileDescriptor=60000, SystemLoadAverage=160, ProcessCount=169, AvailableMemoryMB=16431 2023-07-22T12:32:20,097 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(130): Thread=595 is superior to 500 2023-07-22T12:32:20,097 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] backup.TestFullRestore(55): test full restore on a single table empty table 2023-07-22T12:32:20,101 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a464b6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:32:20,104 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:32:20,105 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:39820, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:32:20,107 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:32:20,108 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:57694, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:32:20,124 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-07-22T12:32:20,129 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:32:20,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029140129 (current time:1690029140129). 2023-07-22T12:32:20,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:32:20,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-07-22T12:32:20,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71b594fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:32:20,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:32:20,137 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:39822, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:32:20,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:32:20,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:32:20,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=68, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:32:20,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 68 2023-07-22T12:32:20,141 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:32:20,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=68 2023-07-22T12:32:20,142 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:32:20,144 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:32:20,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=68 2023-07-22T12:32:20,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=68 2023-07-22T12:32:20,549 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:32:20,549 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33}] 2023-07-22T12:32:20,551 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=69, ppid=68, state=RUNNABLE; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33 2023-07-22T12:32:20,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43639 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=69 2023-07-22T12:32:20,703 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. 2023-07-22T12:32:20,703 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.HRegion(2819): Flushing 697cc5ecf0962f5ab9f4126febf7ce33 2/2 column families, dataSize=1.14 KB heapSize=2.22 KB 2023-07-22T12:32:20,707 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/meta/8f78c6cced064299a30d72a0b4901528 is 169, key is trslm:hdfs://localhost:34745/backupUT\x00test-1690029065625/meta:log-roll-map/1690029139607/Put/seqid=0 2023-07-22T12:32:20,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=68 2023-07-22T12:32:21,112 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=451 B at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/meta/8f78c6cced064299a30d72a0b4901528 2023-07-22T12:32:21,117 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 8f78c6cced064299a30d72a0b4901528 2023-07-22T12:32:21,120 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/session/660347589bb3465a82d9c9138151d6a4 is 452, key is session:backup_1690029128459/session:context/1690029140029/Put/seqid=0 2023-07-22T12:32:21,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=68 2023-07-22T12:32:21,525 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=721 B at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/session/660347589bb3465a82d9c9138151d6a4 2023-07-22T12:32:21,530 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 660347589bb3465a82d9c9138151d6a4 2023-07-22T12:32:21,531 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/meta/8f78c6cced064299a30d72a0b4901528 as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/8f78c6cced064299a30d72a0b4901528 2023-07-22T12:32:21,535 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/8f78c6cced064299a30d72a0b4901528, entries=4, sequenceid=50, filesize=5.5 K 2023-07-22T12:32:21,536 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/session/660347589bb3465a82d9c9138151d6a4 as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/660347589bb3465a82d9c9138151d6a4 2023-07-22T12:32:21,541 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/660347589bb3465a82d9c9138151d6a4, entries=2, sequenceid=50, filesize=5.4 K 2023-07-22T12:32:21,542 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1172, heapSize ~2.19 KB/2240, currentSize=0 B/0 for 697cc5ecf0962f5ab9f4126febf7ce33 in 839ms, sequenceid=50, compaction requested=true 2023-07-22T12:32:21,542 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.HRegion(2520): Flush status journal for 697cc5ecf0962f5ab9f4126febf7ce33: 2023-07-22T12:32:21,542 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. for snapshot_backup_system completed. 2023-07-22T12:32:21,542 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33.' region-info for snapshot=snapshot_backup_system 2023-07-22T12:32:21,542 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:32:21,542 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/b580ac8bece04b07a21605e555bdc696, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/5ea0158e2d574f758c68ee9c20c21191, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/a6d2695c5574440ea495f58a7c66ed03, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/8f78c6cced064299a30d72a0b4901528] hfiles 2023-07-22T12:32:21,542 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (1/4): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/b580ac8bece04b07a21605e555bdc696 for snapshot=snapshot_backup_system 2023-07-22T12:32:21,542 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (2/4): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/5ea0158e2d574f758c68ee9c20c21191 for snapshot=snapshot_backup_system 2023-07-22T12:32:21,542 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (3/4): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/a6d2695c5574440ea495f58a7c66ed03 for snapshot=snapshot_backup_system 2023-07-22T12:32:21,542 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (4/4): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/8f78c6cced064299a30d72a0b4901528 for snapshot=snapshot_backup_system 2023-07-22T12:32:21,542 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/956010b3eaa64b4fa82ddea4557fac2c, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/0704967b71ce4a5cb0af553e6e47addb, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/62b5d2f1465b492ab802e26f5bf32f6b, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/c2fdeacbf4184719a11904adfeaf1fd8, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/660347589bb3465a82d9c9138151d6a4] hfiles 2023-07-22T12:32:21,543 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (1/5): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/956010b3eaa64b4fa82ddea4557fac2c for snapshot=snapshot_backup_system 2023-07-22T12:32:21,543 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (2/5): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/0704967b71ce4a5cb0af553e6e47addb for snapshot=snapshot_backup_system 2023-07-22T12:32:21,543 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (3/5): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/62b5d2f1465b492ab802e26f5bf32f6b for snapshot=snapshot_backup_system 2023-07-22T12:32:21,543 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (4/5): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/c2fdeacbf4184719a11904adfeaf1fd8 for snapshot=snapshot_backup_system 2023-07-22T12:32:21,543 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (5/5): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/660347589bb3465a82d9c9138151d6a4 for snapshot=snapshot_backup_system 2023-07-22T12:32:21,948 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. 2023-07-22T12:32:21,948 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=69 2023-07-22T12:32:21,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster(3994): Remote procedure done, pid=69 2023-07-22T12:32:21,948 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 697cc5ecf0962f5ab9f4126febf7ce33 2023-07-22T12:32:21,950 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=69, ppid=68, state=RUNNABLE; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33 2023-07-22T12:32:21,953 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=69, resume processing ppid=68 2023-07-22T12:32:21,953 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:32:21,953 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=69, ppid=68, state=SUCCESS; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33 in 1.4020 sec 2023-07-22T12:32:21,953 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:32:21,954 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:32:21,954 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-07-22T12:32:21,955 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_backup_system 2023-07-22T12:32:22,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=68 2023-07-22T12:32:22,367 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:32:22,378 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:32:22,379 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_backup_system 2023-07-22T12:32:23,198 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:32:23,198 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 68 2023-07-22T12:32:23,200 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=68, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 3.0590 sec 2023-07-22T12:32:24,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=68 2023-07-22T12:32:24,247 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(115): Backup backup_1690029140101 started at 1690029144247. 2023-07-22T12:32:24,249 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(119): Backup session backup_1690029140101 has been started. 2023-07-22T12:32:24,255 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-07-22T12:32:24,255 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-07-22T12:32:24,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure.ProcedureCoordinator(143): Procedure rolllog was in running list but was completed. Accepting new attempt. 2023-07-22T12:32:24,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-07-22T12:32:24,256 INFO [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-07-22T12:32:24,256 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-07-22T12:32:24,257 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-07-22T12:32:24,257 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-07-22T12:32:24,257 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:32:24,259 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:24,259 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:32:24,259 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:32:24,259 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-07-22T12:32:24,259 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-07-22T12:32:24,260 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-07-22T12:32:24,260 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:32:24,260 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-07-22T12:32:24,260 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-07-22T12:32:24,260 WARN [zk-event-processor-pool-0 {}] procedure.ProcedureMember(133): A completed old subproc rolllog is still present, removing 2023-07-22T12:32:24,260 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-07-22T12:32:24,260 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-07-22T12:32:24,260 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-07-22T12:32:24,261 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-07-22T12:32:24,261 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-07-22T12:32:24,261 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,43639,1690029056327' joining acquired barrier for procedure (rolllog) in zk 2023-07-22T12:32:24,263 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-07-22T12:32:24,263 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:24,263 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:32:24,263 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:32:24,264 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-07-22T12:32:24,264 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:32:24,264 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:32:24,264 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:32:24,264 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:24,265 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:32:24,265 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,43639,1690029056327' joining acquired barrier for procedure 'rolllog' on coordinator 2023-07-22T12:32:24,265 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@43861f78[Count = 0] remaining members to acquire global barrier 2023-07-22T12:32:24,265 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-07-22T12:32:24,265 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-07-22T12:32:24,267 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-07-22T12:32:24,267 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-07-22T12:32:24,267 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-07-22T12:32:24,267 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-07-22T12:32:24,267 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-07-22T12:32:24,268 DEBUG [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:24,268 INFO [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1690029132643 highest: 1690029132643 on jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:24,268 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta:.meta(num 1690029132624) roll requested 2023-07-22T12:32:24,279 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029144268.meta, exclude list is [], retry=0 2023-07-22T12:32:24,283 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK] 2023-07-22T12:32:24,285 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029132624.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029144268.meta 2023-07-22T12:32:24,285 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK]] 2023-07-22T12:32:24,285 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029132624.meta is not closed yet, will try archiving it next time 2023-07-22T12:32:24,285 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C43639%2C1690029056327:(num 1690029132643) roll requested 2023-07-22T12:32:24,288 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029132624.meta to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/oldWALs/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029132624.meta 2023-07-22T12:32:24,297 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029144285, exclude list is [], retry=0 2023-07-22T12:32:24,301 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK] 2023-07-22T12:32:24,303 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029132643 with entries=10, filesize=2.31 KB; new WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029144285 2023-07-22T12:32:24,303 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK]] 2023-07-22T12:32:24,303 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029132643 is not closed yet, will try archiving it next time 2023-07-22T12:32:24,304 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029121076 to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/oldWALs/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029121076 2023-07-22T12:32:24,308 DEBUG [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-07-22T12:32:24,308 INFO [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1690029144285 on jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:24,317 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-07-22T12:32:24,317 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,43639,1690029056327' in zk 2023-07-22T12:32:24,321 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-07-22T12:32:24,321 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:32:24,321 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-07-22T12:32:24,321 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:24,322 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:32:24,322 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:32:24,322 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:32:24,322 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:32:24,323 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:32:24,323 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:24,323 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:32:24,323 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:32:24,324 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:24,324 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,43639,1690029056327': 2023-07-22T12:32:24,324 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,43639,1690029056327' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-07-22T12:32:24,324 INFO [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-07-22T12:32:24,324 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-07-22T12:32:24,324 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-07-22T12:32:24,324 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-07-22T12:32:24,324 INFO [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-07-22T12:32:24,326 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-07-22T12:32:24,326 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-07-22T12:32:24,326 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:32:24,326 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:32:24,326 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:32:24,326 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:32:24,326 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:32:24,327 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:32:24,327 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:32:24,327 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:32:24,327 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:32:24,327 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:32:24,328 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:24,328 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:32:24,328 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:32:24,329 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:24,334 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:32:24,334 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:32:24,334 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:32:24,334 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-07-22T12:32:24,335 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-07-22T12:32:24,335 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:32:24,335 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:32:24,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-07-22T12:32:24,438 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1690029144438_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } 2023-07-22T12:32:24,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029144438 (current time:1690029144438). 2023-07-22T12:32:24,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:32:24,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1690029144438_default_test-1690029065625 VERSION not specified, setting to 2 2023-07-22T12:32:24,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f724bfa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:32:24,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:32:24,448 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:56214, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:32:24,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:32:24,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:32:24,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=70, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1690029144438_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } 2023-07-22T12:32:24,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1690029144438_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 }, snapshot procedure id = 70 2023-07-22T12:32:24,452 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1690029144438_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:32:24,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=70 2023-07-22T12:32:24,452 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1690029144438_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:32:24,454 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1690029144438_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:32:24,459 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1690029144438_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:32:24,459 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE; SnapshotRegionProcedure d1d0c6c0a475cf01e7de961f9001d124}] 2023-07-22T12:32:24,461 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=71, ppid=70, state=RUNNABLE; SnapshotRegionProcedure d1d0c6c0a475cf01e7de961f9001d124 2023-07-22T12:32:24,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=70 2023-07-22T12:32:24,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43639 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=71 2023-07-22T12:32:24,611 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1690029065625,,1690029112094.d1d0c6c0a475cf01e7de961f9001d124. 2023-07-22T12:32:24,612 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] regionserver.HRegion(2520): Flush status journal for d1d0c6c0a475cf01e7de961f9001d124: 2023-07-22T12:32:24,612 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1690029065625,,1690029112094.d1d0c6c0a475cf01e7de961f9001d124. for snapshot_1690029144438_default_test-1690029065625 completed. 2023-07-22T12:32:24,612 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] snapshot.SnapshotManifest(240): Storing 'test-1690029065625,,1690029112094.d1d0c6c0a475cf01e7de961f9001d124.' region-info for snapshot=snapshot_1690029144438_default_test-1690029065625 2023-07-22T12:32:24,612 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:32:24,612 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/d1d0c6c0a475cf01e7de961f9001d124/f/acd648b54cf0414a8319c9fc8c54d368_SeqId_4_] hfiles 2023-07-22T12:32:24,612 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/d1d0c6c0a475cf01e7de961f9001d124/f/acd648b54cf0414a8319c9fc8c54d368_SeqId_4_ for snapshot=snapshot_1690029144438_default_test-1690029065625 2023-07-22T12:32:24,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=70 2023-07-22T12:32:25,018 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1690029065625,,1690029112094.d1d0c6c0a475cf01e7de961f9001d124. 2023-07-22T12:32:25,018 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=71 2023-07-22T12:32:25,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster(3994): Remote procedure done, pid=71 2023-07-22T12:32:25,019 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1690029144438_default_test-1690029065625 on region d1d0c6c0a475cf01e7de961f9001d124 2023-07-22T12:32:25,020 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=71, ppid=70, state=RUNNABLE; SnapshotRegionProcedure d1d0c6c0a475cf01e7de961f9001d124 2023-07-22T12:32:25,022 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=71, resume processing ppid=70 2023-07-22T12:32:25,022 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=71, ppid=70, state=SUCCESS; SnapshotRegionProcedure d1d0c6c0a475cf01e7de961f9001d124 in 562 msec 2023-07-22T12:32:25,022 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1690029144438_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:32:25,023 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1690029144438_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:32:25,023 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1690029144438_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:32:25,024 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1690029144438_default_test-1690029065625 2023-07-22T12:32:25,024 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_1690029144438_default_test-1690029065625 2023-07-22T12:32:25,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=70 2023-07-22T12:32:25,434 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1690029144438_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:32:25,439 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1690029144438_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:32:25,439 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_1690029144438_default_test-1690029065625 to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029144438_default_test-1690029065625 2023-07-22T12:32:25,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=70 2023-07-22T12:32:26,257 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1690029144438_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:32:26,257 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1690029144438_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 }, snapshot procedure id = 70 2023-07-22T12:32:26,259 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=70, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1690029144438_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } in 1.8080 sec 2023-07-22T12:32:26,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=70 2023-07-22T12:32:26,557 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1690029140101 2023-07-22T12:32:26,557 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-07-22T12:32:26,557 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-07-22T12:32:26,557 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1690029140101_test-1690029065625 2023-07-22T12:32:26,557 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1690029144438_default_test-1690029065625 to hdfs://localhost:34745/backupUT/backup_1690029140101/default/test-1690029065625/ 2023-07-22T12:32:26,557 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-07-22T12:32:26,581 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:34745, inputRoot=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947 2023-07-22T12:32:26,581 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,37923,1690029053050_-938248356_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:34745/backupUT/backup_1690029140101/default/test-1690029065625, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:34745/backupUT/backup_1690029140101/default/test-1690029065625/.hbase-snapshot/.tmp/snapshot_1690029144438_default_test-1690029065625 2023-07-22T12:32:26,581 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:34745, inputRoot=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947, snapshotDir=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029144438_default_test-1690029065625. 2023-07-22T12:32:26,589 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029144438_default_test-1690029065625 to hdfs://localhost:34745/backupUT/backup_1690029140101/default/test-1690029065625/.hbase-snapshot/.tmp/snapshot_1690029144438_default_test-1690029065625 2023-07-22T12:32:27,004 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:27,004 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:27,005 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:27,005 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:27,966 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/hadoop-7499673406679218058.jar 2023-07-22T12:32:27,966 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:27,966 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:27,967 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:27,967 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:27,968 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:27,968 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:27,968 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:27,969 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:27,969 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:27,970 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:32:27,970 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-07-22T12:32:27,971 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-07-22T12:32:27,971 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-07-22T12:32:27,971 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-07-22T12:32:27,972 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-07-22T12:32:27,972 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-07-22T12:32:27,973 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-07-22T12:32:27,973 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-07-22T12:32:27,974 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-07-22T12:32:27,974 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-07-22T12:32:27,974 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-07-22T12:32:27,975 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:32:27,976 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:32:27,976 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:32:27,976 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:32:27,977 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:32:27,977 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:32:27,978 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:32:28,045 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1690029144438_default_test-1690029065625' hfile list 2023-07-22T12:32:28,048 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-07-22T12:32:28,117 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029148083/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-07-22T12:32:28,117 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029148083/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-07-22T12:32:28,182 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-07-22T12:32:28,206 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/d1d0c6c0a475cf01e7de961f9001d124/f/acd648b54cf0414a8319c9fc8c54d368_SeqId_4_ output=hdfs://localhost:34745/backupUT/backup_1690029140101/default/test-1690029065625/archive/data/default/test-1690029065625/d1d0c6c0a475cf01e7de961f9001d124/f/acd648b54cf0414a8319c9fc8c54d368_SeqId_4_ 2023-07-22T12:32:28,206 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8724 (8.5 K) time=0sec 8.320M/sec 2023-07-22T12:32:29,167 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-07-22T12:32:29,168 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-07-22T12:32:29,174 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1690029144438_default_test-1690029065625 2023-07-22T12:32:29,175 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1690029144438_default_test-1690029065625 finished. 2023-07-22T12:32:29,175 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(1014): test-1690029065625 2023-07-22T12:32:29,181 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029140101 2023-07-22T12:32:29,181 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-07-22T12:32:29,587 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:34745/backupUT/backup_1690029140101/.backup.manifest 2023-07-22T12:32:29,587 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(358): Backup backup_1690029140101 finished: type=FULL,tablelist=test-1690029065625,targetRootDir=hdfs://localhost:34745/backupUT,startts=1690029144247,completets=1690029149181,bytescopied=0 2023-07-22T12:32:29,587 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-07-22T12:32:29,587 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1690029144438_default_test-1690029065625 2023-07-22T12:32:29,588 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1690029144438_default_test-1690029065625" 2023-07-22T12:32:29,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1690029144438_default_test-1690029065625 2023-07-22T12:32:29,591 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1690029144438_default_test-1690029065625 for backup backup_1690029140101 succeeded. 2023-07-22T12:32:29,592 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-07-22T12:32:29,597 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-07-22T12:32:29,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-07-22T12:32:29,600 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-07-22T12:32:29,602 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-07-22T12:32:29,607 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(376): Backup backup_1690029140101 completed. 2023-07-22T12:32:29,607 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:32:29,617 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] backup.TestFullRestore(61): backup complete 2023-07-22T12:32:29,618 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(101): HBase table table1 does not exist. It will be created during restore process 2023-07-22T12:32:29,619 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34745/backupUT/backup_1690029140101/.backup.manifest 2023-07-22T12:32:29,621 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029140101/.backup.manifest 2023-07-22T12:32:29,621 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34745/backupUT/backup_1690029140101/.backup.manifest 2023-07-22T12:32:29,623 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029140101/.backup.manifest 2023-07-22T12:32:29,623 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(148): Restoring 'test-1690029065625' to 'table1' from full backup image hdfs://localhost:34745/backupUT/backup_1690029140101/default/test-1690029065625 2023-07-22T12:32:29,630 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] util.RestoreTool(487): Creating target table 'table1' 2023-07-22T12:32:29,630 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] util.RestoreTool(391): Parsing region dir: hdfs://localhost:34745/backupUT/backup_1690029140101/default/test-1690029065625/archive/data/default/test-1690029065625/d1d0c6c0a475cf01e7de961f9001d124 2023-07-22T12:32:29,631 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:34745/backupUT/backup_1690029140101/default/test-1690029065625/archive/data/default/test-1690029065625/d1d0c6c0a475cf01e7de961f9001d124/f in region [hdfs://localhost:34745/backupUT/backup_1690029140101/default/test-1690029065625/archive/data/default/test-1690029065625/d1d0c6c0a475cf01e7de961f9001d124] 2023-07-22T12:32:29,634 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] util.RestoreTool(442): Trying to figure out region boundaries hfile=hdfs://localhost:34745/backupUT/backup_1690029140101/default/test-1690029065625/archive/data/default/test-1690029065625/d1d0c6c0a475cf01e7de961f9001d124/f/acd648b54cf0414a8319c9fc8c54d368_SeqId_4_ first=row0 last=row98 2023-07-22T12:32:29,634 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: acd648b54cf0414a8319c9fc8c54d368_SeqId_4_ 2023-07-22T12:32:29,635 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:32:29,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=72, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=table1 2023-07-22T12:32:29,638 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:32:29,638 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "table1" procId is: 72 2023-07-22T12:32:29,639 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:32:29,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=72 2023-07-22T12:32:29,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=72 2023-07-22T12:32:29,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=72 2023-07-22T12:32:30,048 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 0774b9f89a371aa9f52c3a5c68708809, NAME => 'table1,,1690029149635.0774b9f89a371aa9f52c3a5c68708809.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947 2023-07-22T12:32:30,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=72 2023-07-22T12:32:30,454 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(888): Instantiated table1,,1690029149635.0774b9f89a371aa9f52c3a5c68708809.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:32:30,454 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1668): Closing 0774b9f89a371aa9f52c3a5c68708809, disabling compactions & flushes 2023-07-22T12:32:30,454 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1690): Closing region table1,,1690029149635.0774b9f89a371aa9f52c3a5c68708809. 2023-07-22T12:32:30,454 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on table1,,1690029149635.0774b9f89a371aa9f52c3a5c68708809. 2023-07-22T12:32:30,454 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on table1,,1690029149635.0774b9f89a371aa9f52c3a5c68708809. after waiting 0 ms 2023-07-22T12:32:30,454 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region table1,,1690029149635.0774b9f89a371aa9f52c3a5c68708809. 2023-07-22T12:32:30,454 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1909): Closed table1,,1690029149635.0774b9f89a371aa9f52c3a5c68708809. 2023-07-22T12:32:30,454 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1622): Region close journal for 0774b9f89a371aa9f52c3a5c68708809: 2023-07-22T12:32:30,455 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:32:30,455 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"table1,,1690029149635.0774b9f89a371aa9f52c3a5c68708809.","families":{"info":[{"qualifier":"regioninfo","vlen":40,"tag":[],"timestamp":"1690029150455"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690029150455"}]},"ts":"1690029150455"} 2023-07-22T12:32:30,457 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:32:30,457 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:32:30,458 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029150457"}]},"ts":"1690029150457"} 2023-07-22T12:32:30,459 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=ENABLING in hbase:meta 2023-07-22T12:32:30,463 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=table1, region=0774b9f89a371aa9f52c3a5c68708809, ASSIGN}] 2023-07-22T12:32:30,464 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=table1, region=0774b9f89a371aa9f52c3a5c68708809, ASSIGN 2023-07-22T12:32:30,465 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=table1, region=0774b9f89a371aa9f52c3a5c68708809, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,43639,1690029056327; forceNewPlan=false, retain=false 2023-07-22T12:32:30,616 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=73 updating hbase:meta row=0774b9f89a371aa9f52c3a5c68708809, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:30,618 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE; OpenRegionProcedure 0774b9f89a371aa9f52c3a5c68708809, server=jenkins-hbase3.apache.org,43639,1690029056327}] 2023-07-22T12:32:30,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=72 2023-07-22T12:32:30,773 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(130): Open table1,,1690029149635.0774b9f89a371aa9f52c3a5c68708809. 2023-07-22T12:32:30,773 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7254): Opening region: {ENCODED => 0774b9f89a371aa9f52c3a5c68708809, NAME => 'table1,,1690029149635.0774b9f89a371aa9f52c3a5c68708809.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:32:30,773 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:32:30,773 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 0774b9f89a371aa9f52c3a5c68708809 2023-07-22T12:32:30,773 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(888): Instantiated table1,,1690029149635.0774b9f89a371aa9f52c3a5c68708809.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:32:30,773 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7296): checking encryption for 0774b9f89a371aa9f52c3a5c68708809 2023-07-22T12:32:30,773 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7299): checking classloading for 0774b9f89a371aa9f52c3a5c68708809 2023-07-22T12:32:30,775 INFO [StoreOpener-0774b9f89a371aa9f52c3a5c68708809-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 0774b9f89a371aa9f52c3a5c68708809 2023-07-22T12:32:30,776 INFO [StoreOpener-0774b9f89a371aa9f52c3a5c68708809-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0774b9f89a371aa9f52c3a5c68708809 columnFamilyName f 2023-07-22T12:32:30,776 DEBUG [StoreOpener-0774b9f89a371aa9f52c3a5c68708809-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:32:30,776 INFO [StoreOpener-0774b9f89a371aa9f52c3a5c68708809-1 {}] regionserver.HStore(324): Store=0774b9f89a371aa9f52c3a5c68708809/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:32:30,777 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/table1/0774b9f89a371aa9f52c3a5c68708809 2023-07-22T12:32:30,777 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/table1/0774b9f89a371aa9f52c3a5c68708809 2023-07-22T12:32:30,779 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1076): writing seq id for 0774b9f89a371aa9f52c3a5c68708809 2023-07-22T12:32:30,781 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/table1/0774b9f89a371aa9f52c3a5c68708809/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:32:30,781 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1093): Opened 0774b9f89a371aa9f52c3a5c68708809; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9685720960, jitterRate=-0.09794694185256958}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-07-22T12:32:30,782 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(992): Region open journal for 0774b9f89a371aa9f52c3a5c68708809: 2023-07-22T12:32:30,783 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2552): Post open deploy tasks for table1,,1690029149635.0774b9f89a371aa9f52c3a5c68708809., pid=74, masterSystemTime=1690029150769 2023-07-22T12:32:30,785 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2579): Finished post open deploy task for table1,,1690029149635.0774b9f89a371aa9f52c3a5c68708809. 2023-07-22T12:32:30,785 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(158): Opened table1,,1690029149635.0774b9f89a371aa9f52c3a5c68708809. 2023-07-22T12:32:30,785 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=73 updating hbase:meta row=0774b9f89a371aa9f52c3a5c68708809, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:30,787 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=74, resume processing ppid=73 2023-07-22T12:32:30,787 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=74, ppid=73, state=SUCCESS; OpenRegionProcedure 0774b9f89a371aa9f52c3a5c68708809, server=jenkins-hbase3.apache.org,43639,1690029056327 in 168 msec 2023-07-22T12:32:30,789 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=73, resume processing ppid=72 2023-07-22T12:32:30,789 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=73, ppid=72, state=SUCCESS; TransitRegionStateProcedure table=table1, region=0774b9f89a371aa9f52c3a5c68708809, ASSIGN in 324 msec 2023-07-22T12:32:30,789 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:32:30,789 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029150789"}]},"ts":"1690029150789"} 2023-07-22T12:32:30,791 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=ENABLED in hbase:meta 2023-07-22T12:32:30,794 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:32:30,795 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=72, state=SUCCESS; CreateTableProcedure table=table1 in 1.1590 sec 2023-07-22T12:32:31,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=72 2023-07-22T12:32:31,747 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:34745/backupUT/backup_1690029140101/default/test-1690029065625/archive/data/default/test-1690029065625/d1d0c6c0a475cf01e7de961f9001d124 from hbase tables test-1690029065625 to tables table1 2023-07-22T12:32:31,747 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceRestoreJob(72): Restore test-1690029065625 into table1 2023-07-22T12:32:31,749 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-default-table1-1690029151747 from hdfs://localhost:34745/backupUT/backup_1690029140101/default/test-1690029065625/archive/data/default/test-1690029065625/d1d0c6c0a475cf01e7de961f9001d124 2023-07-22T12:32:31,755 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@334b665c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:32:31,756 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:32:31,756 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:36848, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:32:31,757 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-07-22T12:32:31,757 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table table1 2023-07-22T12:32:31,761 DEBUG [hconnection-0x12a9a1af-shared-pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:32:31,762 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:60614, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:32:31,764 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for table1: 2023-07-22T12:32:31,764 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-07-22T12:32:31,764 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/partitions_5a0a77b3-3e63-41ee-bf13-fd9e2f5450c3 2023-07-22T12:32:32,171 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:32,172 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:32,172 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:32,172 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:33,162 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/hadoop-8925135864683302347.jar 2023-07-22T12:32:33,163 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:33,163 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:33,163 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:33,164 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:33,164 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:33,164 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:33,165 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:33,165 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:33,165 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:33,166 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:32:33,167 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-07-22T12:32:33,167 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-07-22T12:32:33,167 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-07-22T12:32:33,168 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-07-22T12:32:33,168 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-07-22T12:32:33,169 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-07-22T12:32:33,169 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-07-22T12:32:33,169 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-07-22T12:32:33,170 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-07-22T12:32:33,170 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-07-22T12:32:33,171 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-07-22T12:32:33,171 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:33,172 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:33,172 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:33,172 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:33,172 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:33,173 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:33,173 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:32:33,174 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: table1 2023-07-22T12:32:33,174 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:32:33,174 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-07-22T12:32:33,174 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-07-22T12:32:33,175 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:32:33,189 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-07-22T12:32:33,225 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_table1 2023-07-22T12:32:33,225 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_table1 Metrics about Tables on a single HBase RegionServer 2023-07-22T12:32:33,287 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029153248/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-07-22T12:32:33,287 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029153248/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-07-22T12:32:33,349 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:34745/backupUT/backup_1690029140101/default/test-1690029065625/archive/data/default/test-1690029065625/d1d0c6c0a475cf01e7de961f9001d124/f/acd648b54cf0414a8319c9fc8c54d368_SeqId_4_ 2023-07-22T12:32:33,354 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-07-22T12:32:33,356 DEBUG [LocalJobRunner Map Task Executor #0 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: acd648b54cf0414a8319c9fc8c54d368_SeqId_4_ 2023-07-22T12:32:33,381 DEBUG [pool-297-thread-1 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29608bcb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:32:33,387 DEBUG [hconnection-0x66c6df5e-metaLookup-shared--pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:32:33,388 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:60616, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:32:33,390 DEBUG [pool-297-thread-1 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:32:33,392 DEBUG [pool-297-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-07-22T12:32:33,392 DEBUG [pool-297-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-07-22T12:32:33,402 DEBUG [pool-297-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-default-table1-1690029151747/_temporary/0/_temporary/attempt_local2018767616_0009_r_000000_0/f/fcf70271858f47e79a0d0c631ba57acb is 37, key is row10/f:q1/1690029068268/Put/seqid=0 2023-07-22T12:32:33,404 WARN [Thread-1345 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:43639] Specified favored nodes: [jenkins-hbase3.apache.org:43639] 2023-07-22T12:32:34,346 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-default-table1-1690029151747 2023-07-22T12:32:34,351 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@be5e9ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:32:34,353 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:32:34,354 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:60632, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:32:34,359 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-default-table1-1690029151747/_SUCCESS 2023-07-22T12:32:34,363 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:32:34,364 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:36862, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:32:34,374 DEBUG [LoadIncrementalHFiles-0 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: fcf70271858f47e79a0d0c631ba57acb 2023-07-22T12:32:34,374 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-default-table1-1690029151747/f/fcf70271858f47e79a0d0c631ba57acb first=Optional[row0] last=Optional[row98] 2023-07-22T12:32:34,375 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=table1,,1690029149635.0774b9f89a371aa9f52c3a5c68708809., hostname=jenkins-hbase3.apache.org,43639,1690029056327, seqNum=2 for row with hfile group [{f,hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-default-table1-1690029151747/f/fcf70271858f47e79a0d0c631ba57acb}] 2023-07-22T12:32:34,380 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-default-table1-1690029151747/f/fcf70271858f47e79a0d0c631ba57acb for inclusion in 0774b9f89a371aa9f52c3a5c68708809/f 2023-07-22T12:32:34,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-07-22T12:32:34,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HStore(625): Region bounds: first= last= 2023-07-22T12:32:34,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: fcf70271858f47e79a0d0c631ba57acb 2023-07-22T12:32:34,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HRegion(2520): Flush status journal for 0774b9f89a371aa9f52c3a5c68708809: 2023-07-22T12:32:34,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-default-table1-1690029151747/f/fcf70271858f47e79a0d0c631ba57acb to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/staging/jenkins__table1__qhhl177g52qe5ro28n4mdidtgd0cjls6o7i3uopmlcvag10iag2voggsb6tva7v6/f/fcf70271858f47e79a0d0c631ba57acb 2023-07-22T12:32:34,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/staging/jenkins__table1__qhhl177g52qe5ro28n4mdidtgd0cjls6o7i3uopmlcvag10iag2voggsb6tva7v6/f/fcf70271858f47e79a0d0c631ba57acb as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/table1/0774b9f89a371aa9f52c3a5c68708809/f/e7d13dbf3796423698087214a9b5dd6a_SeqId_4_ 2023-07-22T12:32:34,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b4da461, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:32:34,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:32:34,394 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:36866, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:32:34,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:32:34,398 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:60648, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:32:34,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:32:34,415 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/staging/jenkins__table1__qhhl177g52qe5ro28n4mdidtgd0cjls6o7i3uopmlcvag10iag2voggsb6tva7v6/f/fcf70271858f47e79a0d0c631ba57acb into 0774b9f89a371aa9f52c3a5c68708809/f as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/table1/0774b9f89a371aa9f52c3a5c68708809/f/e7d13dbf3796423698087214a9b5dd6a_SeqId_4_ - updating store file list. 2023-07-22T12:32:34,420 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/table1/0774b9f89a371aa9f52c3a5c68708809/f/e7d13dbf3796423698087214a9b5dd6a_SeqId_4_ into 0774b9f89a371aa9f52c3a5c68708809/f 2023-07-22T12:32:34,421 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/staging/jenkins__table1__qhhl177g52qe5ro28n4mdidtgd0cjls6o7i3uopmlcvag10iag2voggsb6tva7v6/f/fcf70271858f47e79a0d0c631ba57acb into 0774b9f89a371aa9f52c3a5c68708809/f (new location: hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/table1/0774b9f89a371aa9f52c3a5c68708809/f/e7d13dbf3796423698087214a9b5dd6a_SeqId_4_) 2023-07-22T12:32:34,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/staging/jenkins__table1__qhhl177g52qe5ro28n4mdidtgd0cjls6o7i3uopmlcvag10iag2voggsb6tva7v6/f/fcf70271858f47e79a0d0c631ba57acb 2023-07-22T12:32:34,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51f96abc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:32:34,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:32:34,430 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:36878, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-07-22T12:32:34,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:32:34,434 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:60658, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-07-22T12:32:34,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:32:34,447 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:32:34,447 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-07-22T12:32:34,448 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-07-22T12:32:34,448 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(233): Backup: backup_1690029140101 hdfs://localhost:34745/backupUT/backup_1690029140101/default/test-1690029065625/ 2023-07-22T12:32:34,448 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-07-22T12:32:34,449 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster$12(2663): Client=jenkins//172.31.12.81 disable table1 2023-07-22T12:32:34,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=75, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=table1 2023-07-22T12:32:34,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=75 2023-07-22T12:32:34,452 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029154452"}]},"ts":"1690029154452"} 2023-07-22T12:32:34,453 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=DISABLING in hbase:meta 2023-07-22T12:32:34,455 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(293): Set table1 to state=DISABLING 2023-07-22T12:32:34,456 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=table1, region=0774b9f89a371aa9f52c3a5c68708809, UNASSIGN}] 2023-07-22T12:32:34,457 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=table1, region=0774b9f89a371aa9f52c3a5c68708809, UNASSIGN 2023-07-22T12:32:34,457 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=76 updating hbase:meta row=0774b9f89a371aa9f52c3a5c68708809, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:34,458 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE; CloseRegionProcedure 0774b9f89a371aa9f52c3a5c68708809, server=jenkins-hbase3.apache.org,43639,1690029056327}] 2023-07-22T12:32:34,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=75 2023-07-22T12:32:34,610 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(119): Close 0774b9f89a371aa9f52c3a5c68708809 2023-07-22T12:32:34,610 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1668): Closing 0774b9f89a371aa9f52c3a5c68708809, disabling compactions & flushes 2023-07-22T12:32:34,610 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1690): Closing region table1,,1690029149635.0774b9f89a371aa9f52c3a5c68708809. 2023-07-22T12:32:34,610 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1711): Waiting without time limit for close lock on table1,,1690029149635.0774b9f89a371aa9f52c3a5c68708809. 2023-07-22T12:32:34,610 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1778): Acquired close lock on table1,,1690029149635.0774b9f89a371aa9f52c3a5c68708809. after waiting 0 ms 2023-07-22T12:32:34,610 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1788): Updates disabled for region table1,,1690029149635.0774b9f89a371aa9f52c3a5c68708809. 2023-07-22T12:32:34,612 DEBUG [StoreFileCloser-0774b9f89a371aa9f52c3a5c68708809-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: e7d13dbf3796423698087214a9b5dd6a_SeqId_4_ 2023-07-22T12:32:34,617 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/table1/0774b9f89a371aa9f52c3a5c68708809/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2023-07-22T12:32:34,618 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:32:34,618 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1909): Closed table1,,1690029149635.0774b9f89a371aa9f52c3a5c68708809. 2023-07-22T12:32:34,618 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1622): Region close journal for 0774b9f89a371aa9f52c3a5c68708809: 2023-07-22T12:32:34,619 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(163): Closed 0774b9f89a371aa9f52c3a5c68708809 2023-07-22T12:32:34,620 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=76 updating hbase:meta row=0774b9f89a371aa9f52c3a5c68708809, regionState=CLOSED 2023-07-22T12:32:34,622 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=77, resume processing ppid=76 2023-07-22T12:32:34,622 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=77, ppid=76, state=SUCCESS; CloseRegionProcedure 0774b9f89a371aa9f52c3a5c68708809, server=jenkins-hbase3.apache.org,43639,1690029056327 in 163 msec 2023-07-22T12:32:34,624 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=76, resume processing ppid=75 2023-07-22T12:32:34,624 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=76, ppid=75, state=SUCCESS; TransitRegionStateProcedure table=table1, region=0774b9f89a371aa9f52c3a5c68708809, UNASSIGN in 167 msec 2023-07-22T12:32:34,624 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029154624"}]},"ts":"1690029154624"} 2023-07-22T12:32:34,625 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=DISABLED in hbase:meta 2023-07-22T12:32:34,628 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(305): Set table1 to state=DISABLED 2023-07-22T12:32:34,629 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=75, state=SUCCESS; DisableTableProcedure table=table1 in 179 msec 2023-07-22T12:32:34,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=75 2023-07-22T12:32:34,759 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster$5(2447): Client=jenkins//172.31.12.81 delete table1 2023-07-22T12:32:34,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=78, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=table1 2023-07-22T12:32:34,768 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=78, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=table1 2023-07-22T12:32:34,769 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=78, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=table1 2023-07-22T12:32:34,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=78 2023-07-22T12:32:34,771 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/table1/0774b9f89a371aa9f52c3a5c68708809 2023-07-22T12:32:34,773 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/table1/0774b9f89a371aa9f52c3a5c68708809/f, FileablePath, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/table1/0774b9f89a371aa9f52c3a5c68708809/recovered.edits] 2023-07-22T12:32:34,776 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/table1/0774b9f89a371aa9f52c3a5c68708809/f/e7d13dbf3796423698087214a9b5dd6a_SeqId_4_ to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/archive/data/default/table1/0774b9f89a371aa9f52c3a5c68708809/f/e7d13dbf3796423698087214a9b5dd6a_SeqId_4_ 2023-07-22T12:32:34,779 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/table1/0774b9f89a371aa9f52c3a5c68708809/recovered.edits/6.seqid to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/archive/data/default/table1/0774b9f89a371aa9f52c3a5c68708809/recovered.edits/6.seqid 2023-07-22T12:32:34,779 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/table1/0774b9f89a371aa9f52c3a5c68708809 2023-07-22T12:32:34,779 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(293): Archived table1 regions 2023-07-22T12:32:34,781 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=78, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=table1 2023-07-22T12:32:34,783 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of table1 from hbase:meta 2023-07-22T12:32:34,785 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(387): Removing 'table1' descriptor. 2023-07-22T12:32:34,786 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=78, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=table1 2023-07-22T12:32:34,786 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(377): Removing 'table1' from region states. 2023-07-22T12:32:34,786 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"table1,,1690029149635.0774b9f89a371aa9f52c3a5c68708809.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1690029154786"}]},"ts":"9223372036854775807"} 2023-07-22T12:32:34,787 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-07-22T12:32:34,788 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 0774b9f89a371aa9f52c3a5c68708809, NAME => 'table1,,1690029149635.0774b9f89a371aa9f52c3a5c68708809.', STARTKEY => '', ENDKEY => ''}] 2023-07-22T12:32:34,788 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(381): Marking 'table1' as deleted. 2023-07-22T12:32:34,788 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1690029154788"}]},"ts":"9223372036854775807"} 2023-07-22T12:32:34,789 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1678): Deleted table table1 state from META 2023-07-22T12:32:34,791 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(127): Finished pid=78, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=table1 2023-07-22T12:32:34,792 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=78, state=SUCCESS; DeleteTableProcedure table=table1 in 30 msec 2023-07-22T12:32:34,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=78 2023-07-22T12:32:34,888 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreSingle Thread=594 (was 595), OpenFileDescriptor=670 (was 677), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=140 (was 160), ProcessCount=169 (was 169), AvailableMemoryMB=16256 (was 16431) 2023-07-22T12:32:34,888 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(130): Thread=594 is superior to 500 2023-07-22T12:32:34,902 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreSingleOverwriteCommand Thread=594, OpenFileDescriptor=670, MaxFileDescriptor=60000, SystemLoadAverage=140, ProcessCount=169, AvailableMemoryMB=16256 2023-07-22T12:32:34,902 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(130): Thread=594 is superior to 500 2023-07-22T12:32:34,902 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] backup.TestFullRestore(192): test full restore on a single table empty table: command-line 2023-07-22T12:32:34,907 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63e13a45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:32:34,910 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:32:34,911 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:60666, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:32:34,912 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:32:34,914 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:36882, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:32:34,925 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-07-22T12:32:34,931 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:32:34,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029154931 (current time:1690029154931). 2023-07-22T12:32:34,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:32:34,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-07-22T12:32:34,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7416ace3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:32:34,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:32:34,938 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:60672, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:32:34,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:32:34,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:32:34,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=79, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:32:34,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 79 2023-07-22T12:32:34,941 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:32:34,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=79 2023-07-22T12:32:34,942 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:32:34,944 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:32:35,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=79 2023-07-22T12:32:35,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=79 2023-07-22T12:32:35,350 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:32:35,350 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33}] 2023-07-22T12:32:35,352 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=80, ppid=79, state=RUNNABLE; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33 2023-07-22T12:32:35,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43639 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=80 2023-07-22T12:32:35,503 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. 2023-07-22T12:32:35,503 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.HRegion(2819): Flushing 697cc5ecf0962f5ab9f4126febf7ce33 2/2 column families, dataSize=1.14 KB heapSize=2.22 KB 2023-07-22T12:32:35,507 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/meta/606f117b1e384a54bb6dba5339e90e8f is 169, key is trslm:hdfs://localhost:34745/backupUT\x00test-1690029065625/meta:log-roll-map/1690029149177/Put/seqid=0 2023-07-22T12:32:35,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=79 2023-07-22T12:32:35,912 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=451 B at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/meta/606f117b1e384a54bb6dba5339e90e8f 2023-07-22T12:32:35,917 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 606f117b1e384a54bb6dba5339e90e8f 2023-07-22T12:32:35,920 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/session/2b4c40a0167b43f584d34520d18e9390 is 452, key is session:backup_1690029140101/session:context/1690029149601/Put/seqid=0 2023-07-22T12:32:36,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=79 2023-07-22T12:32:36,316 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-07-22T12:32:36,324 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=721 B at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/session/2b4c40a0167b43f584d34520d18e9390 2023-07-22T12:32:36,330 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 2b4c40a0167b43f584d34520d18e9390 2023-07-22T12:32:36,331 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/meta/606f117b1e384a54bb6dba5339e90e8f as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/606f117b1e384a54bb6dba5339e90e8f 2023-07-22T12:32:36,337 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/606f117b1e384a54bb6dba5339e90e8f, entries=4, sequenceid=61, filesize=5.5 K 2023-07-22T12:32:36,338 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/session/2b4c40a0167b43f584d34520d18e9390 as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/2b4c40a0167b43f584d34520d18e9390 2023-07-22T12:32:36,344 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/2b4c40a0167b43f584d34520d18e9390, entries=2, sequenceid=61, filesize=5.4 K 2023-07-22T12:32:36,346 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1172, heapSize ~2.19 KB/2240, currentSize=0 B/0 for 697cc5ecf0962f5ab9f4126febf7ce33 in 842ms, sequenceid=61, compaction requested=true 2023-07-22T12:32:36,346 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.HRegion(2520): Flush status journal for 697cc5ecf0962f5ab9f4126febf7ce33: 2023-07-22T12:32:36,346 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. for snapshot_backup_system completed. 2023-07-22T12:32:36,346 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33.' region-info for snapshot=snapshot_backup_system 2023-07-22T12:32:36,346 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:32:36,346 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/b580ac8bece04b07a21605e555bdc696, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/5ea0158e2d574f758c68ee9c20c21191, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/a6d2695c5574440ea495f58a7c66ed03, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/8f78c6cced064299a30d72a0b4901528, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/606f117b1e384a54bb6dba5339e90e8f] hfiles 2023-07-22T12:32:36,346 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (1/5): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/b580ac8bece04b07a21605e555bdc696 for snapshot=snapshot_backup_system 2023-07-22T12:32:36,346 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (2/5): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/5ea0158e2d574f758c68ee9c20c21191 for snapshot=snapshot_backup_system 2023-07-22T12:32:36,346 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (3/5): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/a6d2695c5574440ea495f58a7c66ed03 for snapshot=snapshot_backup_system 2023-07-22T12:32:36,346 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (4/5): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/8f78c6cced064299a30d72a0b4901528 for snapshot=snapshot_backup_system 2023-07-22T12:32:36,346 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (5/5): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/606f117b1e384a54bb6dba5339e90e8f for snapshot=snapshot_backup_system 2023-07-22T12:32:36,346 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/956010b3eaa64b4fa82ddea4557fac2c, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/0704967b71ce4a5cb0af553e6e47addb, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/62b5d2f1465b492ab802e26f5bf32f6b, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/c2fdeacbf4184719a11904adfeaf1fd8, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/660347589bb3465a82d9c9138151d6a4, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/2b4c40a0167b43f584d34520d18e9390] hfiles 2023-07-22T12:32:36,346 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (1/6): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/956010b3eaa64b4fa82ddea4557fac2c for snapshot=snapshot_backup_system 2023-07-22T12:32:36,346 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (2/6): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/0704967b71ce4a5cb0af553e6e47addb for snapshot=snapshot_backup_system 2023-07-22T12:32:36,346 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (3/6): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/62b5d2f1465b492ab802e26f5bf32f6b for snapshot=snapshot_backup_system 2023-07-22T12:32:36,347 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (4/6): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/c2fdeacbf4184719a11904adfeaf1fd8 for snapshot=snapshot_backup_system 2023-07-22T12:32:36,347 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (5/6): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/660347589bb3465a82d9c9138151d6a4 for snapshot=snapshot_backup_system 2023-07-22T12:32:36,347 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (6/6): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/2b4c40a0167b43f584d34520d18e9390 for snapshot=snapshot_backup_system 2023-07-22T12:32:36,357 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. 2023-07-22T12:32:36,358 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=80 2023-07-22T12:32:36,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster(3994): Remote procedure done, pid=80 2023-07-22T12:32:36,358 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 697cc5ecf0962f5ab9f4126febf7ce33 2023-07-22T12:32:36,359 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=80, ppid=79, state=RUNNABLE; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33 2023-07-22T12:32:36,362 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=80, resume processing ppid=79 2023-07-22T12:32:36,362 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:32:36,362 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=80, ppid=79, state=SUCCESS; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33 in 1.0100 sec 2023-07-22T12:32:36,363 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:32:36,363 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:32:36,363 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-07-22T12:32:36,364 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_backup_system 2023-07-22T12:32:36,777 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:32:36,786 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:32:36,787 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_backup_system 2023-07-22T12:32:37,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=79 2023-07-22T12:32:37,603 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:32:37,603 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 79 2023-07-22T12:32:37,604 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=79, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.6630 sec 2023-07-22T12:32:39,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=79 2023-07-22T12:32:39,048 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(115): Backup backup_1690029154907 started at 1690029159048. 2023-07-22T12:32:39,050 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(119): Backup session backup_1690029154907 has been started. 2023-07-22T12:32:39,055 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-07-22T12:32:39,056 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-07-22T12:32:39,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-07-22T12:32:39,056 INFO [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-07-22T12:32:39,056 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-07-22T12:32:39,057 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-07-22T12:32:39,057 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-07-22T12:32:39,057 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:32:39,061 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:39,061 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:32:39,061 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:32:39,061 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-07-22T12:32:39,061 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-07-22T12:32:39,062 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-07-22T12:32:39,062 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:32:39,062 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-07-22T12:32:39,062 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-07-22T12:32:39,062 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-07-22T12:32:39,062 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-07-22T12:32:39,062 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-07-22T12:32:39,063 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-07-22T12:32:39,063 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-07-22T12:32:39,063 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,43639,1690029056327' joining acquired barrier for procedure (rolllog) in zk 2023-07-22T12:32:39,066 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-07-22T12:32:39,066 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:39,066 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:32:39,066 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:32:39,066 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-07-22T12:32:39,066 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:32:39,066 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:32:39,067 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:32:39,067 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:39,067 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:32:39,067 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,43639,1690029056327' joining acquired barrier for procedure 'rolllog' on coordinator 2023-07-22T12:32:39,067 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@6f790ec7[Count = 0] remaining members to acquire global barrier 2023-07-22T12:32:39,067 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-07-22T12:32:39,068 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-07-22T12:32:39,069 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-07-22T12:32:39,069 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-07-22T12:32:39,069 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-07-22T12:32:39,070 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-07-22T12:32:39,070 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-07-22T12:32:39,070 DEBUG [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:39,070 INFO [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1690029144285 highest: 1690029144285 on jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:39,070 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta:.meta(num 1690029144268) roll requested 2023-07-22T12:32:39,083 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029159070.meta, exclude list is [], retry=0 2023-07-22T12:32:39,090 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK] 2023-07-22T12:32:39,092 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029144268.meta with entries=12, filesize=2.73 KB; new WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029159070.meta 2023-07-22T12:32:39,092 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK]] 2023-07-22T12:32:39,092 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029144268.meta is not closed yet, will try archiving it next time 2023-07-22T12:32:39,092 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C43639%2C1690029056327:(num 1690029144285) roll requested 2023-07-22T12:32:39,104 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029159092, exclude list is [], retry=0 2023-07-22T12:32:39,107 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK] 2023-07-22T12:32:39,108 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029144285 with entries=13, filesize=3.09 KB; new WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029159092 2023-07-22T12:32:39,108 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK]] 2023-07-22T12:32:39,108 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029144285 is not closed yet, will try archiving it next time 2023-07-22T12:32:39,109 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029132643 to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/oldWALs/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029132643 2023-07-22T12:32:39,110 DEBUG [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-07-22T12:32:39,110 INFO [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1690029159092 on jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:39,111 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029144285 not finished, retry = 0 2023-07-22T12:32:39,119 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-07-22T12:32:39,119 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,43639,1690029056327' in zk 2023-07-22T12:32:39,121 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-07-22T12:32:39,121 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:39,121 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:32:39,121 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:32:39,121 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-07-22T12:32:39,122 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:32:39,122 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:32:39,123 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:32:39,123 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:32:39,123 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:39,123 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:32:39,123 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:32:39,124 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:39,124 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,43639,1690029056327': 2023-07-22T12:32:39,124 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,43639,1690029056327' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-07-22T12:32:39,124 INFO [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-07-22T12:32:39,124 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-07-22T12:32:39,124 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-07-22T12:32:39,124 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-07-22T12:32:39,124 INFO [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-07-22T12:32:39,126 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-07-22T12:32:39,126 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-07-22T12:32:39,126 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:32:39,126 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:32:39,126 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:32:39,126 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:32:39,126 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:32:39,126 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:32:39,127 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:32:39,127 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:32:39,127 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:32:39,127 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:32:39,128 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:39,128 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:32:39,128 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:32:39,129 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:39,131 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:32:39,131 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:32:39,131 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:32:39,131 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-07-22T12:32:39,132 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-07-22T12:32:39,132 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:32:39,132 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:32:39,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-07-22T12:32:39,236 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1690029159236_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } 2023-07-22T12:32:39,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029159236 (current time:1690029159236). 2023-07-22T12:32:39,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:32:39,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1690029159236_default_test-1690029065625 VERSION not specified, setting to 2 2023-07-22T12:32:39,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@706fd158, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:32:39,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:32:39,249 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:60676, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:32:39,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:32:39,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:32:39,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=81, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1690029159236_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } 2023-07-22T12:32:39,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1690029159236_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 }, snapshot procedure id = 81 2023-07-22T12:32:39,253 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1690029159236_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:32:39,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=81 2023-07-22T12:32:39,254 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1690029159236_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:32:39,255 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1690029159236_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:32:39,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=81 2023-07-22T12:32:39,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=81 2023-07-22T12:32:39,661 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1690029159236_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:32:39,661 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE; SnapshotRegionProcedure d1d0c6c0a475cf01e7de961f9001d124}] 2023-07-22T12:32:39,663 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=82, ppid=81, state=RUNNABLE; SnapshotRegionProcedure d1d0c6c0a475cf01e7de961f9001d124 2023-07-22T12:32:39,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43639 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=82 2023-07-22T12:32:39,815 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1690029065625,,1690029112094.d1d0c6c0a475cf01e7de961f9001d124. 2023-07-22T12:32:39,815 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] regionserver.HRegion(2520): Flush status journal for d1d0c6c0a475cf01e7de961f9001d124: 2023-07-22T12:32:39,815 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1690029065625,,1690029112094.d1d0c6c0a475cf01e7de961f9001d124. for snapshot_1690029159236_default_test-1690029065625 completed. 2023-07-22T12:32:39,815 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] snapshot.SnapshotManifest(240): Storing 'test-1690029065625,,1690029112094.d1d0c6c0a475cf01e7de961f9001d124.' region-info for snapshot=snapshot_1690029159236_default_test-1690029065625 2023-07-22T12:32:39,815 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:32:39,815 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/d1d0c6c0a475cf01e7de961f9001d124/f/acd648b54cf0414a8319c9fc8c54d368_SeqId_4_] hfiles 2023-07-22T12:32:39,815 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/d1d0c6c0a475cf01e7de961f9001d124/f/acd648b54cf0414a8319c9fc8c54d368_SeqId_4_ for snapshot=snapshot_1690029159236_default_test-1690029065625 2023-07-22T12:32:39,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=81 2023-07-22T12:32:40,221 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1690029065625,,1690029112094.d1d0c6c0a475cf01e7de961f9001d124. 2023-07-22T12:32:40,221 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=82 2023-07-22T12:32:40,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster(3994): Remote procedure done, pid=82 2023-07-22T12:32:40,221 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1690029159236_default_test-1690029065625 on region d1d0c6c0a475cf01e7de961f9001d124 2023-07-22T12:32:40,223 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=82, ppid=81, state=RUNNABLE; SnapshotRegionProcedure d1d0c6c0a475cf01e7de961f9001d124 2023-07-22T12:32:40,224 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=82, resume processing ppid=81 2023-07-22T12:32:40,225 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=82, ppid=81, state=SUCCESS; SnapshotRegionProcedure d1d0c6c0a475cf01e7de961f9001d124 in 562 msec 2023-07-22T12:32:40,225 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1690029159236_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:32:40,225 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1690029159236_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:32:40,226 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1690029159236_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:32:40,226 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1690029159236_default_test-1690029065625 2023-07-22T12:32:40,226 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_1690029159236_default_test-1690029065625 2023-07-22T12:32:40,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=81 2023-07-22T12:32:40,635 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1690029159236_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:32:40,641 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1690029159236_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:32:40,641 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_1690029159236_default_test-1690029065625 to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029159236_default_test-1690029065625 2023-07-22T12:32:41,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=81 2023-07-22T12:32:41,458 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1690029159236_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:32:41,458 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1690029159236_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 }, snapshot procedure id = 81 2023-07-22T12:32:41,459 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=81, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1690029159236_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } in 2.2070 sec 2023-07-22T12:32:43,225 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_table1 2023-07-22T12:32:43,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=81 2023-07-22T12:32:43,361 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1690029154907 2023-07-22T12:32:43,361 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-07-22T12:32:43,361 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-07-22T12:32:43,361 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1690029154907_test-1690029065625 2023-07-22T12:32:43,361 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1690029159236_default_test-1690029065625 to hdfs://localhost:34745/backupUT/backup_1690029154907/default/test-1690029065625/ 2023-07-22T12:32:43,361 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-07-22T12:32:43,384 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:34745, inputRoot=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947 2023-07-22T12:32:43,384 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,37923,1690029053050_-959268898_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:34745/backupUT/backup_1690029154907/default/test-1690029065625, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:34745/backupUT/backup_1690029154907/default/test-1690029065625/.hbase-snapshot/.tmp/snapshot_1690029159236_default_test-1690029065625 2023-07-22T12:32:43,384 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:34745, inputRoot=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947, snapshotDir=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029159236_default_test-1690029065625. 2023-07-22T12:32:43,391 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029159236_default_test-1690029065625 to hdfs://localhost:34745/backupUT/backup_1690029154907/default/test-1690029065625/.hbase-snapshot/.tmp/snapshot_1690029159236_default_test-1690029065625 2023-07-22T12:32:43,806 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:43,806 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:43,807 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:43,807 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:44,770 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/hadoop-2968908538821842602.jar 2023-07-22T12:32:44,771 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:44,771 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:44,771 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:44,772 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:44,772 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:44,772 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:44,773 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:44,773 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:44,773 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:44,774 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:32:44,775 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-07-22T12:32:44,775 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-07-22T12:32:44,775 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-07-22T12:32:44,776 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-07-22T12:32:44,776 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-07-22T12:32:44,776 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-07-22T12:32:44,777 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-07-22T12:32:44,777 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-07-22T12:32:44,778 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-07-22T12:32:44,778 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-07-22T12:32:44,778 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-07-22T12:32:44,779 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:32:44,779 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:32:44,780 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:32:44,780 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:32:44,781 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:32:44,781 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:32:44,782 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:32:44,827 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1690029159236_default_test-1690029065625' hfile list 2023-07-22T12:32:44,829 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-07-22T12:32:44,897 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029164864/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-07-22T12:32:44,897 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029164864/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-07-22T12:32:44,973 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-07-22T12:32:44,997 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/d1d0c6c0a475cf01e7de961f9001d124/f/acd648b54cf0414a8319c9fc8c54d368_SeqId_4_ output=hdfs://localhost:34745/backupUT/backup_1690029154907/default/test-1690029065625/archive/data/default/test-1690029065625/d1d0c6c0a475cf01e7de961f9001d124/f/acd648b54cf0414a8319c9fc8c54d368_SeqId_4_ 2023-07-22T12:32:44,997 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8724 (8.5 K) time=0sec 4.160M/sec 2023-07-22T12:32:45,955 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-07-22T12:32:45,956 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-07-22T12:32:45,964 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1690029159236_default_test-1690029065625 2023-07-22T12:32:45,964 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1690029159236_default_test-1690029065625 finished. 2023-07-22T12:32:45,964 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(1014): test-1690029065625 2023-07-22T12:32:45,971 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029154907 2023-07-22T12:32:45,971 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-07-22T12:32:46,376 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:34745/backupUT/backup_1690029154907/.backup.manifest 2023-07-22T12:32:46,376 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(358): Backup backup_1690029154907 finished: type=FULL,tablelist=test-1690029065625,targetRootDir=hdfs://localhost:34745/backupUT,startts=1690029159048,completets=1690029165971,bytescopied=0 2023-07-22T12:32:46,376 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-07-22T12:32:46,376 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1690029159236_default_test-1690029065625 2023-07-22T12:32:46,379 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1690029159236_default_test-1690029065625" 2023-07-22T12:32:46,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1690029159236_default_test-1690029065625 2023-07-22T12:32:46,382 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1690029159236_default_test-1690029065625 for backup backup_1690029154907 succeeded. 2023-07-22T12:32:46,383 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-07-22T12:32:46,387 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-07-22T12:32:46,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-07-22T12:32:46,390 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-07-22T12:32:46,392 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-07-22T12:32:46,397 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(376): Backup backup_1690029154907 completed. 2023-07-22T12:32:46,397 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:32:46,407 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] backup.TestFullRestore(196): backup complete 2023-07-22T12:32:46,407 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] backup.RestoreDriver(101): Found -overwrite option in restore command, will overwrite to existing table if any in the restore target 2023-07-22T12:32:46,415 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f800000, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:32:46,416 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:32:46,418 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:36266, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:32:46,420 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34745/backupUT/backup_1690029154907/.backup.manifest 2023-07-22T12:32:46,423 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029154907/.backup.manifest 2023-07-22T12:32:46,423 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34745/backupUT/backup_1690029154907/.backup.manifest 2023-07-22T12:32:46,425 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029154907/.backup.manifest 2023-07-22T12:32:46,425 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(148): Restoring 'test-1690029065625' to 'test-1690029065625' from full backup image hdfs://localhost:34745/backupUT/backup_1690029154907/default/test-1690029065625 2023-07-22T12:32:46,432 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] util.RestoreTool(476): Truncating exising target table 'test-1690029065625', preserving region splits 2023-07-22T12:32:46,432 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:32:46,434 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:35728, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:32:46,435 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster$12(2663): Client=jenkins//172.31.12.81 disable test-1690029065625 2023-07-22T12:32:46,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=83, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=test-1690029065625 2023-07-22T12:32:46,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=83 2023-07-22T12:32:46,439 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1690029065625","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029166439"}]},"ts":"1690029166439"} 2023-07-22T12:32:46,440 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1690029065625, state=DISABLING in hbase:meta 2023-07-22T12:32:46,443 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(293): Set test-1690029065625 to state=DISABLING 2023-07-22T12:32:46,444 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=test-1690029065625, region=d1d0c6c0a475cf01e7de961f9001d124, UNASSIGN}] 2023-07-22T12:32:46,445 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=test-1690029065625, region=d1d0c6c0a475cf01e7de961f9001d124, UNASSIGN 2023-07-22T12:32:46,446 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=84 updating hbase:meta row=d1d0c6c0a475cf01e7de961f9001d124, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:46,447 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE; CloseRegionProcedure d1d0c6c0a475cf01e7de961f9001d124, server=jenkins-hbase3.apache.org,43639,1690029056327}] 2023-07-22T12:32:46,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=83 2023-07-22T12:32:46,599 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(119): Close d1d0c6c0a475cf01e7de961f9001d124 2023-07-22T12:32:46,599 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1668): Closing d1d0c6c0a475cf01e7de961f9001d124, disabling compactions & flushes 2023-07-22T12:32:46,599 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1690): Closing region test-1690029065625,,1690029112094.d1d0c6c0a475cf01e7de961f9001d124. 2023-07-22T12:32:46,599 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1690029065625,,1690029112094.d1d0c6c0a475cf01e7de961f9001d124. 2023-07-22T12:32:46,599 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1778): Acquired close lock on test-1690029065625,,1690029112094.d1d0c6c0a475cf01e7de961f9001d124. after waiting 0 ms 2023-07-22T12:32:46,599 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1788): Updates disabled for region test-1690029065625,,1690029112094.d1d0c6c0a475cf01e7de961f9001d124. 2023-07-22T12:32:46,601 DEBUG [StoreFileCloser-d1d0c6c0a475cf01e7de961f9001d124-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: acd648b54cf0414a8319c9fc8c54d368_SeqId_4_ 2023-07-22T12:32:46,605 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/d1d0c6c0a475cf01e7de961f9001d124/recovered.edits/10.seqid, newMaxSeqId=10, maxSeqId=1 2023-07-22T12:32:46,605 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:32:46,606 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1909): Closed test-1690029065625,,1690029112094.d1d0c6c0a475cf01e7de961f9001d124. 2023-07-22T12:32:46,606 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1622): Region close journal for d1d0c6c0a475cf01e7de961f9001d124: 2023-07-22T12:32:46,607 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(163): Closed d1d0c6c0a475cf01e7de961f9001d124 2023-07-22T12:32:46,608 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=84 updating hbase:meta row=d1d0c6c0a475cf01e7de961f9001d124, regionState=CLOSED 2023-07-22T12:32:46,610 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=85, resume processing ppid=84 2023-07-22T12:32:46,610 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=85, ppid=84, state=SUCCESS; CloseRegionProcedure d1d0c6c0a475cf01e7de961f9001d124, server=jenkins-hbase3.apache.org,43639,1690029056327 in 162 msec 2023-07-22T12:32:46,611 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=84, resume processing ppid=83 2023-07-22T12:32:46,611 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=84, ppid=83, state=SUCCESS; TransitRegionStateProcedure table=test-1690029065625, region=d1d0c6c0a475cf01e7de961f9001d124, UNASSIGN in 166 msec 2023-07-22T12:32:46,612 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1690029065625","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029166612"}]},"ts":"1690029166612"} 2023-07-22T12:32:46,613 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1690029065625, state=DISABLED in hbase:meta 2023-07-22T12:32:46,614 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(305): Set test-1690029065625 to state=DISABLED 2023-07-22T12:32:46,616 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=83, state=SUCCESS; DisableTableProcedure table=test-1690029065625 in 180 msec 2023-07-22T12:32:46,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=83 2023-07-22T12:32:46,741 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster$6(2479): Client=jenkins//172.31.12.81 truncate test-1690029065625 2023-07-22T12:32:46,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=86, state=RUNNABLE:TRUNCATE_TABLE_PRE_OPERATION; TruncateTableProcedure (table=test-1690029065625 preserveSplits=true) 2023-07-22T12:32:46,744 DEBUG [PEWorker-1 {}] procedure.TruncateTableProcedure(87): waiting for 'test-1690029065625' regions in transition 2023-07-22T12:32:46,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=86 2023-07-22T12:32:46,746 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/d1d0c6c0a475cf01e7de961f9001d124 2023-07-22T12:32:46,748 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/d1d0c6c0a475cf01e7de961f9001d124/f, FileablePath, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/d1d0c6c0a475cf01e7de961f9001d124/recovered.edits] 2023-07-22T12:32:46,752 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/d1d0c6c0a475cf01e7de961f9001d124/f/acd648b54cf0414a8319c9fc8c54d368_SeqId_4_ to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/archive/data/default/test-1690029065625/d1d0c6c0a475cf01e7de961f9001d124/f/acd648b54cf0414a8319c9fc8c54d368_SeqId_4_ 2023-07-22T12:32:46,755 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/d1d0c6c0a475cf01e7de961f9001d124/recovered.edits/10.seqid to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/archive/data/default/test-1690029065625/d1d0c6c0a475cf01e7de961f9001d124/recovered.edits/10.seqid 2023-07-22T12:32:46,755 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/d1d0c6c0a475cf01e7de961f9001d124 2023-07-22T12:32:46,755 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(293): Archived test-1690029065625 regions 2023-07-22T12:32:46,759 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of test-1690029065625 from hbase:meta 2023-07-22T12:32:46,761 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(387): Removing 'test-1690029065625' descriptor. 2023-07-22T12:32:46,761 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(377): Removing 'test-1690029065625' from region states. 2023-07-22T12:32:46,762 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"test-1690029065625,,1690029112094.d1d0c6c0a475cf01e7de961f9001d124.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1690029166762"}]},"ts":"9223372036854775807"} 2023-07-22T12:32:46,763 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-07-22T12:32:46,763 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => d1d0c6c0a475cf01e7de961f9001d124, NAME => 'test-1690029065625,,1690029112094.d1d0c6c0a475cf01e7de961f9001d124.', STARTKEY => '', ENDKEY => ''}] 2023-07-22T12:32:46,763 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(381): Marking 'test-1690029065625' as deleted. 2023-07-22T12:32:46,763 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"test-1690029065625","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1690029166763"}]},"ts":"9223372036854775807"} 2023-07-22T12:32:46,765 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1678): Deleted table test-1690029065625 state from META 2023-07-22T12:32:46,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=86 2023-07-22T12:32:47,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=86 2023-07-22T12:32:47,173 INFO [RegionOpenAndInit-test-1690029065625-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => baf4f0d945c66abea73108272203bc6e, NAME => 'test-1690029065625,,1690029166757.baf4f0d945c66abea73108272203bc6e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1690029065625', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947 2023-07-22T12:32:47,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=86 2023-07-22T12:32:47,579 DEBUG [RegionOpenAndInit-test-1690029065625-pool-0 {}] regionserver.HRegion(888): Instantiated test-1690029065625,,1690029166757.baf4f0d945c66abea73108272203bc6e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:32:47,579 DEBUG [RegionOpenAndInit-test-1690029065625-pool-0 {}] regionserver.HRegion(1668): Closing baf4f0d945c66abea73108272203bc6e, disabling compactions & flushes 2023-07-22T12:32:47,579 INFO [RegionOpenAndInit-test-1690029065625-pool-0 {}] regionserver.HRegion(1690): Closing region test-1690029065625,,1690029166757.baf4f0d945c66abea73108272203bc6e. 2023-07-22T12:32:47,579 DEBUG [RegionOpenAndInit-test-1690029065625-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1690029065625,,1690029166757.baf4f0d945c66abea73108272203bc6e. 2023-07-22T12:32:47,579 DEBUG [RegionOpenAndInit-test-1690029065625-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on test-1690029065625,,1690029166757.baf4f0d945c66abea73108272203bc6e. after waiting 0 ms 2023-07-22T12:32:47,579 DEBUG [RegionOpenAndInit-test-1690029065625-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region test-1690029065625,,1690029166757.baf4f0d945c66abea73108272203bc6e. 2023-07-22T12:32:47,579 INFO [RegionOpenAndInit-test-1690029065625-pool-0 {}] regionserver.HRegion(1909): Closed test-1690029065625,,1690029166757.baf4f0d945c66abea73108272203bc6e. 2023-07-22T12:32:47,579 DEBUG [RegionOpenAndInit-test-1690029065625-pool-0 {}] regionserver.HRegion(1622): Region close journal for baf4f0d945c66abea73108272203bc6e: 2023-07-22T12:32:47,580 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"test-1690029065625,,1690029166757.baf4f0d945c66abea73108272203bc6e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1690029167580"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690029167580"}]},"ts":"1690029167580"} 2023-07-22T12:32:47,582 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:32:47,582 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1690029065625","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029167582"}]},"ts":"1690029167582"} 2023-07-22T12:32:47,583 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1690029065625, state=ENABLING in hbase:meta 2023-07-22T12:32:47,588 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1690029065625, region=baf4f0d945c66abea73108272203bc6e, ASSIGN}] 2023-07-22T12:32:47,590 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=87, ppid=86, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1690029065625, region=baf4f0d945c66abea73108272203bc6e, ASSIGN 2023-07-22T12:32:47,590 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=87, ppid=86, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=test-1690029065625, region=baf4f0d945c66abea73108272203bc6e, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,43639,1690029056327; forceNewPlan=false, retain=false 2023-07-22T12:32:47,742 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=87 updating hbase:meta row=baf4f0d945c66abea73108272203bc6e, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:47,743 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE; OpenRegionProcedure baf4f0d945c66abea73108272203bc6e, server=jenkins-hbase3.apache.org,43639,1690029056327}] 2023-07-22T12:32:47,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=86 2023-07-22T12:32:47,898 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(130): Open test-1690029065625,,1690029166757.baf4f0d945c66abea73108272203bc6e. 2023-07-22T12:32:47,898 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7254): Opening region: {ENCODED => baf4f0d945c66abea73108272203bc6e, NAME => 'test-1690029065625,,1690029166757.baf4f0d945c66abea73108272203bc6e.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:32:47,898 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:32:47,898 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1690029065625 baf4f0d945c66abea73108272203bc6e 2023-07-22T12:32:47,898 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(888): Instantiated test-1690029065625,,1690029166757.baf4f0d945c66abea73108272203bc6e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:32:47,898 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7296): checking encryption for baf4f0d945c66abea73108272203bc6e 2023-07-22T12:32:47,898 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7299): checking classloading for baf4f0d945c66abea73108272203bc6e 2023-07-22T12:32:47,900 INFO [StoreOpener-baf4f0d945c66abea73108272203bc6e-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region baf4f0d945c66abea73108272203bc6e 2023-07-22T12:32:47,901 INFO [StoreOpener-baf4f0d945c66abea73108272203bc6e-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region baf4f0d945c66abea73108272203bc6e columnFamilyName f 2023-07-22T12:32:47,901 DEBUG [StoreOpener-baf4f0d945c66abea73108272203bc6e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:32:47,901 INFO [StoreOpener-baf4f0d945c66abea73108272203bc6e-1 {}] regionserver.HStore(324): Store=baf4f0d945c66abea73108272203bc6e/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:32:47,902 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/baf4f0d945c66abea73108272203bc6e 2023-07-22T12:32:47,902 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/baf4f0d945c66abea73108272203bc6e 2023-07-22T12:32:47,904 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1076): writing seq id for baf4f0d945c66abea73108272203bc6e 2023-07-22T12:32:47,906 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/baf4f0d945c66abea73108272203bc6e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:32:47,906 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1093): Opened baf4f0d945c66abea73108272203bc6e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10253551520, jitterRate=-0.045063599944114685}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-07-22T12:32:47,906 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(992): Region open journal for baf4f0d945c66abea73108272203bc6e: 2023-07-22T12:32:47,907 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2552): Post open deploy tasks for test-1690029065625,,1690029166757.baf4f0d945c66abea73108272203bc6e., pid=88, masterSystemTime=1690029167895 2023-07-22T12:32:47,908 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2579): Finished post open deploy task for test-1690029065625,,1690029166757.baf4f0d945c66abea73108272203bc6e. 2023-07-22T12:32:47,908 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(158): Opened test-1690029065625,,1690029166757.baf4f0d945c66abea73108272203bc6e. 2023-07-22T12:32:47,909 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=87 updating hbase:meta row=baf4f0d945c66abea73108272203bc6e, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:47,911 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=88, resume processing ppid=87 2023-07-22T12:32:47,911 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=88, ppid=87, state=SUCCESS; OpenRegionProcedure baf4f0d945c66abea73108272203bc6e, server=jenkins-hbase3.apache.org,43639,1690029056327 in 167 msec 2023-07-22T12:32:47,912 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=87, resume processing ppid=86 2023-07-22T12:32:47,912 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=87, ppid=86, state=SUCCESS; TransitRegionStateProcedure table=test-1690029065625, region=baf4f0d945c66abea73108272203bc6e, ASSIGN in 323 msec 2023-07-22T12:32:47,912 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1690029065625","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029167912"}]},"ts":"1690029167912"} 2023-07-22T12:32:47,913 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1690029065625, state=ENABLED in hbase:meta 2023-07-22T12:32:47,915 DEBUG [PEWorker-2 {}] procedure.TruncateTableProcedure(145): truncate 'test-1690029065625' completed 2023-07-22T12:32:47,916 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=86, state=SUCCESS; TruncateTableProcedure (table=test-1690029065625 preserveSplits=true) in 1.1740 sec 2023-07-22T12:32:48,774 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-07-22T12:32:48,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=86 2023-07-22T12:32:48,855 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:34745/backupUT/backup_1690029154907/default/test-1690029065625/archive/data/default/test-1690029065625/d1d0c6c0a475cf01e7de961f9001d124 from hbase tables test-1690029065625 to tables test-1690029065625 2023-07-22T12:32:48,855 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceRestoreJob(72): Restore test-1690029065625 into test-1690029065625 2023-07-22T12:32:48,857 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-default-test-1690029065625-1690029168855 from hdfs://localhost:34745/backupUT/backup_1690029154907/default/test-1690029065625/archive/data/default/test-1690029065625/d1d0c6c0a475cf01e7de961f9001d124 2023-07-22T12:32:48,862 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29bb7bd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:32:48,863 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:32:48,864 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:35736, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:32:48,865 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-07-22T12:32:48,865 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table test-1690029065625 2023-07-22T12:32:48,868 DEBUG [hconnection-0x785c76cb-shared-pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:32:48,870 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:36268, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:32:48,871 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for test-1690029065625: 2023-07-22T12:32:48,871 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-07-22T12:32:48,871 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/partitions_44c7e924-ac79-40ec-8ebc-8cec435c0388 2023-07-22T12:32:49,282 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:49,282 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:49,283 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:49,283 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:50,239 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/hadoop-7593311612498898737.jar 2023-07-22T12:32:50,239 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:50,240 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:50,240 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:50,240 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:50,241 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:50,241 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:50,242 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:50,242 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:50,242 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:50,243 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:32:50,244 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-07-22T12:32:50,244 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-07-22T12:32:50,244 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-07-22T12:32:50,245 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-07-22T12:32:50,245 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-07-22T12:32:50,245 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-07-22T12:32:50,246 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-07-22T12:32:50,246 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-07-22T12:32:50,247 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-07-22T12:32:50,247 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-07-22T12:32:50,247 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-07-22T12:32:50,248 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:50,248 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:50,249 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:50,249 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:50,249 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:50,249 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:32:50,250 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:32:50,250 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: test-1690029065625 2023-07-22T12:32:50,250 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:32:50,250 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-07-22T12:32:50,251 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-07-22T12:32:50,251 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:32:50,262 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-07-22T12:32:50,336 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029170302/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-07-22T12:32:50,336 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029170302/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-07-22T12:32:50,385 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:34745/backupUT/backup_1690029154907/default/test-1690029065625/archive/data/default/test-1690029065625/d1d0c6c0a475cf01e7de961f9001d124/f/acd648b54cf0414a8319c9fc8c54d368_SeqId_4_ 2023-07-22T12:32:50,389 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-07-22T12:32:50,391 DEBUG [LocalJobRunner Map Task Executor #0 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: acd648b54cf0414a8319c9fc8c54d368_SeqId_4_ 2023-07-22T12:32:50,409 DEBUG [pool-329-thread-1 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1460fd27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:32:50,411 DEBUG [hconnection-0x5710de54-metaLookup-shared--pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:32:50,413 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:51110, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:32:50,416 DEBUG [pool-329-thread-1 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:32:50,416 DEBUG [pool-329-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-07-22T12:32:50,416 DEBUG [pool-329-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-07-22T12:32:50,422 DEBUG [pool-329-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-default-test-1690029065625-1690029168855/_temporary/0/_temporary/attempt_local903443202_0011_r_000000_0/f/ad782a64e9dd4973aa74f9f5db34b524 is 37, key is row10/f:q1/1690029068268/Put/seqid=0 2023-07-22T12:32:50,424 WARN [Thread-1502 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:43639] Specified favored nodes: [jenkins-hbase3.apache.org:43639] 2023-07-22T12:32:51,384 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-default-test-1690029065625-1690029168855 2023-07-22T12:32:51,389 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7420df74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:32:51,390 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:32:51,391 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:51126, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:32:51,397 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-default-test-1690029065625-1690029168855/_SUCCESS 2023-07-22T12:32:51,401 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:32:51,403 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:34618, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:32:51,412 DEBUG [LoadIncrementalHFiles-0 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: ad782a64e9dd4973aa74f9f5db34b524 2023-07-22T12:32:51,412 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-default-test-1690029065625-1690029168855/f/ad782a64e9dd4973aa74f9f5db34b524 first=Optional[row0] last=Optional[row98] 2023-07-22T12:32:51,413 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=test-1690029065625,,1690029166757.baf4f0d945c66abea73108272203bc6e., hostname=jenkins-hbase3.apache.org,43639,1690029056327, seqNum=2 for row with hfile group [{f,hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-default-test-1690029065625-1690029168855/f/ad782a64e9dd4973aa74f9f5db34b524}] 2023-07-22T12:32:51,421 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-default-test-1690029065625-1690029168855/f/ad782a64e9dd4973aa74f9f5db34b524 for inclusion in baf4f0d945c66abea73108272203bc6e/f 2023-07-22T12:32:51,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-07-22T12:32:51,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HStore(625): Region bounds: first= last= 2023-07-22T12:32:51,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: ad782a64e9dd4973aa74f9f5db34b524 2023-07-22T12:32:51,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HRegion(2520): Flush status journal for baf4f0d945c66abea73108272203bc6e: 2023-07-22T12:32:51,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-default-test-1690029065625-1690029168855/f/ad782a64e9dd4973aa74f9f5db34b524 to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/staging/jenkins__test-1690029065625__tiqdcutvp2bea1tteccce56gfkl224d1nlq2ndt8af8r4465r9sim913pm3a7ouo/f/ad782a64e9dd4973aa74f9f5db34b524 2023-07-22T12:32:51,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/staging/jenkins__test-1690029065625__tiqdcutvp2bea1tteccce56gfkl224d1nlq2ndt8af8r4465r9sim913pm3a7ouo/f/ad782a64e9dd4973aa74f9f5db34b524 as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/baf4f0d945c66abea73108272203bc6e/f/28a90c4c400f435e9fc04a59e5431a5a_SeqId_4_ 2023-07-22T12:32:51,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@607dc516, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:32:51,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:32:51,434 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:34630, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:32:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:32:51,439 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:51140, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:32:51,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] impl.BackupSystemTable(408): write bulk load descriptor to backup test-1690029065625 with 1 entries 2023-07-22T12:32:51,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] impl.BackupSystemTable(1582): writing raw bulk path hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/baf4f0d945c66abea73108272203bc6e/f/28a90c4c400f435e9fc04a59e5431a5a_SeqId_4_ for test-1690029065625 baf4f0d945c66abea73108272203bc6e 2023-07-22T12:32:51,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] impl.BackupSystemTable(415): written 1 rows for bulk load of test-1690029065625 2023-07-22T12:32:51,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:32:51,454 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/staging/jenkins__test-1690029065625__tiqdcutvp2bea1tteccce56gfkl224d1nlq2ndt8af8r4465r9sim913pm3a7ouo/f/ad782a64e9dd4973aa74f9f5db34b524 into baf4f0d945c66abea73108272203bc6e/f as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/baf4f0d945c66abea73108272203bc6e/f/28a90c4c400f435e9fc04a59e5431a5a_SeqId_4_ - updating store file list. 2023-07-22T12:32:51,459 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/baf4f0d945c66abea73108272203bc6e/f/28a90c4c400f435e9fc04a59e5431a5a_SeqId_4_ into baf4f0d945c66abea73108272203bc6e/f 2023-07-22T12:32:51,459 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/staging/jenkins__test-1690029065625__tiqdcutvp2bea1tteccce56gfkl224d1nlq2ndt8af8r4465r9sim913pm3a7ouo/f/ad782a64e9dd4973aa74f9f5db34b524 into baf4f0d945c66abea73108272203bc6e/f (new location: hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/baf4f0d945c66abea73108272203bc6e/f/28a90c4c400f435e9fc04a59e5431a5a_SeqId_4_) 2023-07-22T12:32:51,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/staging/jenkins__test-1690029065625__tiqdcutvp2bea1tteccce56gfkl224d1nlq2ndt8af8r4465r9sim913pm3a7ouo/f/ad782a64e9dd4973aa74f9f5db34b524 2023-07-22T12:32:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7826872a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:32:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:32:51,466 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:34640, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-07-22T12:32:51,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:32:51,471 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:51144, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-07-22T12:32:51,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] impl.BackupSystemTable(388): write bulk load descriptor to backup test-1690029065625 with 1 entries 2023-07-22T12:32:51,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] impl.BackupSystemTable(1503): writing done bulk path hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/baf4f0d945c66abea73108272203bc6e/f/28a90c4c400f435e9fc04a59e5431a5a_SeqId_4_ for test-1690029065625 baf4f0d945c66abea73108272203bc6e 2023-07-22T12:32:51,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] impl.BackupSystemTable(394): written 1 rows for bulk load of test-1690029065625 2023-07-22T12:32:51,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:32:51,489 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:32:51,489 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-07-22T12:32:51,489 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-07-22T12:32:51,490 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(233): Backup: backup_1690029154907 hdfs://localhost:34745/backupUT/backup_1690029154907/default/test-1690029065625/ 2023-07-22T12:32:51,490 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-07-22T12:32:51,490 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:32:51,509 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreSingleOverwriteCommand Thread=604 (was 594) - Thread LEAK? -, OpenFileDescriptor=691 (was 670) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=124 (was 140), ProcessCount=169 (was 169), AvailableMemoryMB=16236 (was 16256) 2023-07-22T12:32:51,509 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(130): Thread=604 is superior to 500 2023-07-22T12:32:51,526 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreMultipleDNECommand Thread=604, OpenFileDescriptor=691, MaxFileDescriptor=60000, SystemLoadAverage=124, ProcessCount=169, AvailableMemoryMB=16235 2023-07-22T12:32:51,526 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(130): Thread=604 is superior to 500 2023-07-22T12:32:51,527 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] backup.TestFullRestore(322): test restore fails on multiple tables that do not exist: command-line 2023-07-22T12:32:51,531 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cb3c420, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:32:51,535 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:32:51,536 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:51150, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:32:51,538 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:32:51,539 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:34642, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:32:51,551 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-07-22T12:32:51,556 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:32:51,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029171556 (current time:1690029171556). 2023-07-22T12:32:51,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:32:51,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-07-22T12:32:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1de00c41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:32:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:32:51,563 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:51162, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:32:51,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:32:51,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:32:51,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=89, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:32:51,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 89 2023-07-22T12:32:51,567 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:32:51,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=89 2023-07-22T12:32:51,567 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:32:51,569 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:32:51,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=89 2023-07-22T12:32:51,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=89 2023-07-22T12:32:51,977 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:32:51,977 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33}] 2023-07-22T12:32:51,979 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=90, ppid=89, state=RUNNABLE; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33 2023-07-22T12:32:52,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43639 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=90 2023-07-22T12:32:52,131 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. 2023-07-22T12:32:52,132 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.HRegion(2819): Flushing 697cc5ecf0962f5ab9f4126febf7ce33 2/2 column families, dataSize=1.14 KB heapSize=2.22 KB 2023-07-22T12:32:52,135 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/meta/15bf4f9b1b65467c96713e751468dc6c is 169, key is trslm:hdfs://localhost:34745/backupUT\x00test-1690029065625/meta:log-roll-map/1690029165966/Put/seqid=0 2023-07-22T12:32:52,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=89 2023-07-22T12:32:52,540 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=451 B at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/meta/15bf4f9b1b65467c96713e751468dc6c 2023-07-22T12:32:52,546 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 15bf4f9b1b65467c96713e751468dc6c 2023-07-22T12:32:52,550 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/session/1550986ca81f443894a286cada73692c is 452, key is session:backup_1690029154907/session:context/1690029166390/Put/seqid=0 2023-07-22T12:32:52,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=89 2023-07-22T12:32:52,955 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=721 B at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/session/1550986ca81f443894a286cada73692c 2023-07-22T12:32:52,960 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 1550986ca81f443894a286cada73692c 2023-07-22T12:32:52,961 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/meta/15bf4f9b1b65467c96713e751468dc6c as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/15bf4f9b1b65467c96713e751468dc6c 2023-07-22T12:32:52,966 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/15bf4f9b1b65467c96713e751468dc6c, entries=4, sequenceid=72, filesize=5.5 K 2023-07-22T12:32:52,967 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/session/1550986ca81f443894a286cada73692c as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/1550986ca81f443894a286cada73692c 2023-07-22T12:32:52,972 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/1550986ca81f443894a286cada73692c, entries=2, sequenceid=72, filesize=5.4 K 2023-07-22T12:32:52,973 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1172, heapSize ~2.19 KB/2240, currentSize=0 B/0 for 697cc5ecf0962f5ab9f4126febf7ce33 in 842ms, sequenceid=72, compaction requested=true 2023-07-22T12:32:52,973 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.HRegion(2520): Flush status journal for 697cc5ecf0962f5ab9f4126febf7ce33: 2023-07-22T12:32:52,973 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. for snapshot_backup_system completed. 2023-07-22T12:32:52,973 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33.' region-info for snapshot=snapshot_backup_system 2023-07-22T12:32:52,973 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:32:52,973 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/b580ac8bece04b07a21605e555bdc696, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/5ea0158e2d574f758c68ee9c20c21191, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/a6d2695c5574440ea495f58a7c66ed03, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/8f78c6cced064299a30d72a0b4901528, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/606f117b1e384a54bb6dba5339e90e8f, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/15bf4f9b1b65467c96713e751468dc6c] hfiles 2023-07-22T12:32:52,973 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (1/6): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/b580ac8bece04b07a21605e555bdc696 for snapshot=snapshot_backup_system 2023-07-22T12:32:52,973 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (2/6): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/5ea0158e2d574f758c68ee9c20c21191 for snapshot=snapshot_backup_system 2023-07-22T12:32:52,973 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (3/6): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/a6d2695c5574440ea495f58a7c66ed03 for snapshot=snapshot_backup_system 2023-07-22T12:32:52,973 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (4/6): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/8f78c6cced064299a30d72a0b4901528 for snapshot=snapshot_backup_system 2023-07-22T12:32:52,973 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (5/6): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/606f117b1e384a54bb6dba5339e90e8f for snapshot=snapshot_backup_system 2023-07-22T12:32:52,973 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (6/6): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/15bf4f9b1b65467c96713e751468dc6c for snapshot=snapshot_backup_system 2023-07-22T12:32:52,973 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/956010b3eaa64b4fa82ddea4557fac2c, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/0704967b71ce4a5cb0af553e6e47addb, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/62b5d2f1465b492ab802e26f5bf32f6b, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/c2fdeacbf4184719a11904adfeaf1fd8, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/660347589bb3465a82d9c9138151d6a4, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/2b4c40a0167b43f584d34520d18e9390, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/1550986ca81f443894a286cada73692c] hfiles 2023-07-22T12:32:52,973 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (1/7): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/956010b3eaa64b4fa82ddea4557fac2c for snapshot=snapshot_backup_system 2023-07-22T12:32:52,974 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (2/7): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/0704967b71ce4a5cb0af553e6e47addb for snapshot=snapshot_backup_system 2023-07-22T12:32:52,974 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (3/7): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/62b5d2f1465b492ab802e26f5bf32f6b for snapshot=snapshot_backup_system 2023-07-22T12:32:52,974 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (4/7): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/c2fdeacbf4184719a11904adfeaf1fd8 for snapshot=snapshot_backup_system 2023-07-22T12:32:52,974 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (5/7): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/660347589bb3465a82d9c9138151d6a4 for snapshot=snapshot_backup_system 2023-07-22T12:32:52,974 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (6/7): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/2b4c40a0167b43f584d34520d18e9390 for snapshot=snapshot_backup_system 2023-07-22T12:32:52,974 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (7/7): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/1550986ca81f443894a286cada73692c for snapshot=snapshot_backup_system 2023-07-22T12:32:53,378 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. 2023-07-22T12:32:53,378 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=90 2023-07-22T12:32:53,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster(3994): Remote procedure done, pid=90 2023-07-22T12:32:53,379 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 697cc5ecf0962f5ab9f4126febf7ce33 2023-07-22T12:32:53,380 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=90, ppid=89, state=RUNNABLE; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33 2023-07-22T12:32:53,382 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=90, resume processing ppid=89 2023-07-22T12:32:53,382 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:32:53,382 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=90, ppid=89, state=SUCCESS; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33 in 1.4040 sec 2023-07-22T12:32:53,383 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:32:53,383 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:32:53,383 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-07-22T12:32:53,384 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_backup_system 2023-07-22T12:32:53,509 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test-1690029065625' 2023-07-22T12:32:53,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=89 2023-07-22T12:32:53,793 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:32:53,804 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:32:53,804 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_backup_system 2023-07-22T12:32:54,623 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:32:54,623 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 89 2023-07-22T12:32:54,623 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=89, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 3.0590 sec 2023-07-22T12:32:55,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=89 2023-07-22T12:32:55,673 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(115): Backup backup_1690029171531 started at 1690029175673. 2023-07-22T12:32:55,674 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(119): Backup session backup_1690029171531 has been started. 2023-07-22T12:32:55,680 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-07-22T12:32:55,680 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-07-22T12:32:55,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-07-22T12:32:55,680 INFO [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-07-22T12:32:55,680 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-07-22T12:32:55,681 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-07-22T12:32:55,681 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-07-22T12:32:55,682 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:32:55,683 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:55,683 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:32:55,683 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:32:55,683 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-07-22T12:32:55,683 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-07-22T12:32:55,684 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-07-22T12:32:55,684 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:32:55,684 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-07-22T12:32:55,684 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-07-22T12:32:55,684 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-07-22T12:32:55,684 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-07-22T12:32:55,684 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-07-22T12:32:55,685 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-07-22T12:32:55,685 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-07-22T12:32:55,685 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,43639,1690029056327' joining acquired barrier for procedure (rolllog) in zk 2023-07-22T12:32:55,687 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-07-22T12:32:55,687 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:55,687 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:32:55,687 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:32:55,687 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-07-22T12:32:55,687 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:32:55,688 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:32:55,688 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:32:55,688 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:55,689 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:32:55,689 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,43639,1690029056327' joining acquired barrier for procedure 'rolllog' on coordinator 2023-07-22T12:32:55,689 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@557ced60[Count = 0] remaining members to acquire global barrier 2023-07-22T12:32:55,689 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-07-22T12:32:55,689 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-07-22T12:32:55,690 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-07-22T12:32:55,690 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-07-22T12:32:55,690 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-07-22T12:32:55,690 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-07-22T12:32:55,691 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-07-22T12:32:55,691 DEBUG [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:55,691 INFO [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1690029159092 highest: 1690029159092 on jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:55,691 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta:.meta(num 1690029159070) roll requested 2023-07-22T12:32:55,702 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029175691.meta, exclude list is [], retry=0 2023-07-22T12:32:55,706 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK] 2023-07-22T12:32:55,708 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029159070.meta with entries=12, filesize=3.07 KB; new WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029175691.meta 2023-07-22T12:32:55,708 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK]] 2023-07-22T12:32:55,708 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029159070.meta is not closed yet, will try archiving it next time 2023-07-22T12:32:55,708 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C43639%2C1690029056327:(num 1690029159092) roll requested 2023-07-22T12:32:55,719 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029175708, exclude list is [], retry=0 2023-07-22T12:32:55,722 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK] 2023-07-22T12:32:55,723 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029159092 with entries=15, filesize=4.76 KB; new WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029175708 2023-07-22T12:32:55,723 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK]] 2023-07-22T12:32:55,723 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029159092 is not closed yet, will try archiving it next time 2023-07-22T12:32:55,724 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029144285 to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/oldWALs/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029144285 2023-07-22T12:32:55,731 DEBUG [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-07-22T12:32:55,731 INFO [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1690029175708 on jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:55,738 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-07-22T12:32:55,738 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,43639,1690029056327' in zk 2023-07-22T12:32:55,741 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-07-22T12:32:55,741 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:55,741 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:32:55,741 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:32:55,741 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-07-22T12:32:55,742 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:32:55,742 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:32:55,743 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:32:55,743 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:32:55,743 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:55,743 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:32:55,744 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:32:55,744 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:55,744 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,43639,1690029056327': 2023-07-22T12:32:55,744 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,43639,1690029056327' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-07-22T12:32:55,744 INFO [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-07-22T12:32:55,745 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-07-22T12:32:55,745 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-07-22T12:32:55,745 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-07-22T12:32:55,745 INFO [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-07-22T12:32:55,746 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-07-22T12:32:55,746 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-07-22T12:32:55,746 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:32:55,746 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:32:55,746 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:32:55,746 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:32:55,746 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:32:55,747 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:32:55,747 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:32:55,747 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:32:55,747 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:32:55,747 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:32:55,748 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:55,748 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:32:55,748 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:32:55,748 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:32:55,752 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:32:55,752 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:32:55,752 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:32:55,752 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-07-22T12:32:55,752 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-07-22T12:32:55,753 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:32:55,753 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:32:55,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-07-22T12:32:55,856 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1690029175856_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } 2023-07-22T12:32:55,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029175856 (current time:1690029175856). 2023-07-22T12:32:55,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:32:55,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1690029175856_ns2_test-16900290656251 VERSION not specified, setting to 2 2023-07-22T12:32:55,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bec027, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:32:55,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:32:55,863 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:51168, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:32:55,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:32:55,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:32:55,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=91, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1690029175856_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } 2023-07-22T12:32:55,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1690029175856_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 }, snapshot procedure id = 91 2023-07-22T12:32:55,866 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1690029175856_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:32:55,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=91 2023-07-22T12:32:55,867 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1690029175856_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:32:55,869 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1690029175856_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:32:55,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=91 2023-07-22T12:32:56,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=91 2023-07-22T12:32:56,279 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1690029175856_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:32:56,280 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE; SnapshotRegionProcedure b8196e3466217e5359c3acacaaeb9e61}] 2023-07-22T12:32:56,281 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=92, ppid=91, state=RUNNABLE; SnapshotRegionProcedure b8196e3466217e5359c3acacaaeb9e61 2023-07-22T12:32:56,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43639 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=92 2023-07-22T12:32:56,433 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-16900290656251,,1690029096058.b8196e3466217e5359c3acacaaeb9e61. 2023-07-22T12:32:56,433 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] regionserver.HRegion(2520): Flush status journal for b8196e3466217e5359c3acacaaeb9e61: 2023-07-22T12:32:56,433 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-16900290656251,,1690029096058.b8196e3466217e5359c3acacaaeb9e61. for snapshot_1690029175856_ns2_test-16900290656251 completed. 2023-07-22T12:32:56,433 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] snapshot.SnapshotManifest(240): Storing 'ns2:test-16900290656251,,1690029096058.b8196e3466217e5359c3acacaaeb9e61.' region-info for snapshot=snapshot_1690029175856_ns2_test-16900290656251 2023-07-22T12:32:56,433 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:32:56,433 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/b8196e3466217e5359c3acacaaeb9e61/f/82851155b50f47edb054b7e8d9f06090_SeqId_4_] hfiles 2023-07-22T12:32:56,433 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/b8196e3466217e5359c3acacaaeb9e61/f/82851155b50f47edb054b7e8d9f06090_SeqId_4_ for snapshot=snapshot_1690029175856_ns2_test-16900290656251 2023-07-22T12:32:56,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=91 2023-07-22T12:32:56,839 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-16900290656251,,1690029096058.b8196e3466217e5359c3acacaaeb9e61. 2023-07-22T12:32:56,839 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=92 2023-07-22T12:32:56,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster(3994): Remote procedure done, pid=92 2023-07-22T12:32:56,840 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1690029175856_ns2_test-16900290656251 on region b8196e3466217e5359c3acacaaeb9e61 2023-07-22T12:32:56,841 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=92, ppid=91, state=RUNNABLE; SnapshotRegionProcedure b8196e3466217e5359c3acacaaeb9e61 2023-07-22T12:32:56,842 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=92, resume processing ppid=91 2023-07-22T12:32:56,843 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1690029175856_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:32:56,843 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=92, ppid=91, state=SUCCESS; SnapshotRegionProcedure b8196e3466217e5359c3acacaaeb9e61 in 561 msec 2023-07-22T12:32:56,843 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1690029175856_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:32:56,843 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1690029175856_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:32:56,843 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1690029175856_ns2_test-16900290656251 2023-07-22T12:32:56,844 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_1690029175856_ns2_test-16900290656251 2023-07-22T12:32:56,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=91 2023-07-22T12:32:57,253 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1690029175856_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:32:57,258 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1690029175856_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:32:57,258 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_1690029175856_ns2_test-16900290656251 to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029175856_ns2_test-16900290656251 2023-07-22T12:32:57,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=91 2023-07-22T12:32:58,076 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1690029175856_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:32:58,077 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1690029175856_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 }, snapshot procedure id = 91 2023-07-22T12:32:58,078 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=91, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1690029175856_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } in 2.2130 sec 2023-07-22T12:32:58,818 DEBUG [regionserver/jenkins-hbase3:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.3076923076923077, tune throughput to 65.38 MB/second 2023-07-22T12:32:59,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=91 2023-07-22T12:32:59,974 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1690029179973_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } 2023-07-22T12:32:59,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029179974 (current time:1690029179974). 2023-07-22T12:32:59,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:32:59,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1690029179973_ns3_test-16900290656252 VERSION not specified, setting to 2 2023-07-22T12:32:59,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@231ae224, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:32:59,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:32:59,982 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:51172, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:32:59,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:32:59,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:32:59,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=93, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1690029179973_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } 2023-07-22T12:32:59,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1690029179973_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 }, snapshot procedure id = 93 2023-07-22T12:32:59,985 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1690029179973_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:32:59,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=93 2023-07-22T12:32:59,986 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1690029179973_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:32:59,988 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1690029179973_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:33:00,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=93 2023-07-22T12:33:00,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=93 2023-07-22T12:33:00,393 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1690029179973_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:33:00,394 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE; SnapshotRegionProcedure aefea80783ea312734eb5d79ad00b2f1}] 2023-07-22T12:33:00,395 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=94, ppid=93, state=RUNNABLE; SnapshotRegionProcedure aefea80783ea312734eb5d79ad00b2f1 2023-07-22T12:33:00,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43639 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=94 2023-07-22T12:33:00,547 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=94}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-16900290656252,,1690029100393.aefea80783ea312734eb5d79ad00b2f1. 2023-07-22T12:33:00,548 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=94}] regionserver.HRegion(2520): Flush status journal for aefea80783ea312734eb5d79ad00b2f1: 2023-07-22T12:33:00,548 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=94}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-16900290656252,,1690029100393.aefea80783ea312734eb5d79ad00b2f1. for snapshot_1690029179973_ns3_test-16900290656252 completed. 2023-07-22T12:33:00,549 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=94}] snapshot.SnapshotManifest(240): Storing 'ns3:test-16900290656252,,1690029100393.aefea80783ea312734eb5d79ad00b2f1.' region-info for snapshot=snapshot_1690029179973_ns3_test-16900290656252 2023-07-22T12:33:00,549 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=94}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:33:00,549 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=94}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-07-22T12:33:00,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=93 2023-07-22T12:33:00,954 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=94}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-16900290656252,,1690029100393.aefea80783ea312734eb5d79ad00b2f1. 2023-07-22T12:33:00,954 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=94 2023-07-22T12:33:00,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster(3994): Remote procedure done, pid=94 2023-07-22T12:33:00,955 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1690029179973_ns3_test-16900290656252 on region aefea80783ea312734eb5d79ad00b2f1 2023-07-22T12:33:00,956 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=94, ppid=93, state=RUNNABLE; SnapshotRegionProcedure aefea80783ea312734eb5d79ad00b2f1 2023-07-22T12:33:00,959 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=94, resume processing ppid=93 2023-07-22T12:33:00,959 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=94, ppid=93, state=SUCCESS; SnapshotRegionProcedure aefea80783ea312734eb5d79ad00b2f1 in 564 msec 2023-07-22T12:33:00,959 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1690029179973_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:33:00,959 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1690029179973_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:33:00,960 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1690029179973_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:33:00,960 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1690029179973_ns3_test-16900290656252 2023-07-22T12:33:00,961 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_1690029179973_ns3_test-16900290656252 2023-07-22T12:33:00,975 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1690029179973_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:33:00,980 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1690029179973_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:33:00,981 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_1690029179973_ns3_test-16900290656252 to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029179973_ns3_test-16900290656252 2023-07-22T12:33:01,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=93 2023-07-22T12:33:01,806 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1690029179973_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:33:01,806 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1690029179973_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 }, snapshot procedure id = 93 2023-07-22T12:33:01,807 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=93, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1690029179973_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } in 1.8230 sec 2023-07-22T12:33:02,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=93 2023-07-22T12:33:02,091 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1690029171531 2023-07-22T12:33:02,091 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-07-22T12:33:02,091 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2023-07-22T12:33:02,091 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1690029171531_ns2:test-16900290656251 2023-07-22T12:33:02,091 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1690029175856_ns2_test-16900290656251 to hdfs://localhost:34745/backupUT/backup_1690029171531/ns2/test-16900290656251/ 2023-07-22T12:33:02,091 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-07-22T12:33:02,114 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:34745, inputRoot=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947 2023-07-22T12:33:02,114 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,37923,1690029053050_1685171413_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:34745/backupUT/backup_1690029171531/ns2/test-16900290656251, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:34745/backupUT/backup_1690029171531/ns2/test-16900290656251/.hbase-snapshot/.tmp/snapshot_1690029175856_ns2_test-16900290656251 2023-07-22T12:33:02,114 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:34745, inputRoot=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947, snapshotDir=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029175856_ns2_test-16900290656251. 2023-07-22T12:33:02,121 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029175856_ns2_test-16900290656251 to hdfs://localhost:34745/backupUT/backup_1690029171531/ns2/test-16900290656251/.hbase-snapshot/.tmp/snapshot_1690029175856_ns2_test-16900290656251 2023-07-22T12:33:02,534 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:02,534 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:02,535 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:02,535 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:03,493 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/hadoop-3415768344607053700.jar 2023-07-22T12:33:03,493 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:03,494 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:03,494 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:03,494 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:03,495 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:03,495 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:03,495 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:03,496 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:03,496 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:03,497 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:33:03,497 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-07-22T12:33:03,498 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-07-22T12:33:03,498 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-07-22T12:33:03,498 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-07-22T12:33:03,499 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-07-22T12:33:03,499 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-07-22T12:33:03,499 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-07-22T12:33:03,500 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-07-22T12:33:03,500 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-07-22T12:33:03,501 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-07-22T12:33:03,501 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-07-22T12:33:03,502 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:33:03,502 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:33:03,503 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:33:03,503 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:33:03,503 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:33:03,504 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:33:03,504 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:33:03,546 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1690029175856_ns2_test-16900290656251' hfile list 2023-07-22T12:33:03,548 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-07-22T12:33:03,618 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029183583/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-07-22T12:33:03,618 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029183583/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-07-22T12:33:03,696 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-07-22T12:33:03,719 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/b8196e3466217e5359c3acacaaeb9e61/f/82851155b50f47edb054b7e8d9f06090_SeqId_4_ output=hdfs://localhost:34745/backupUT/backup_1690029171531/ns2/test-16900290656251/archive/data/ns2/test-16900290656251/b8196e3466217e5359c3acacaaeb9e61/f/82851155b50f47edb054b7e8d9f06090_SeqId_4_ 2023-07-22T12:33:03,719 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8725 (8.5 K) time=0sec 8.321M/sec 2023-07-22T12:33:04,681 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-07-22T12:33:04,682 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-07-22T12:33:04,688 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1690029175856_ns2_test-16900290656251 2023-07-22T12:33:04,688 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1690029175856_ns2_test-16900290656251 finished. 2023-07-22T12:33:04,688 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1690029171531_ns3:test-16900290656252 2023-07-22T12:33:04,688 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1690029179973_ns3_test-16900290656252 to hdfs://localhost:34745/backupUT/backup_1690029171531/ns3/test-16900290656252/ 2023-07-22T12:33:04,688 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-07-22T12:33:04,722 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:34745, inputRoot=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947 2023-07-22T12:33:04,723 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,37923,1690029053050_1356055591_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:34745/backupUT/backup_1690029171531/ns3/test-16900290656252, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:34745/backupUT/backup_1690029171531/ns3/test-16900290656252/.hbase-snapshot/.tmp/snapshot_1690029179973_ns3_test-16900290656252 2023-07-22T12:33:04,723 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:34745, inputRoot=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947, snapshotDir=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029179973_ns3_test-16900290656252. 2023-07-22T12:33:04,728 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029179973_ns3_test-16900290656252 to hdfs://localhost:34745/backupUT/backup_1690029171531/ns3/test-16900290656252/.hbase-snapshot/.tmp/snapshot_1690029179973_ns3_test-16900290656252 2023-07-22T12:33:05,144 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:05,144 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:05,145 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:05,145 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:06,115 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/hadoop-4723160750994258995.jar 2023-07-22T12:33:06,115 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:06,115 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:06,116 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:06,116 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:06,117 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:06,117 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:06,117 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:06,118 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:06,118 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:06,119 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:33:06,119 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-07-22T12:33:06,120 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-07-22T12:33:06,120 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-07-22T12:33:06,120 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-07-22T12:33:06,121 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-07-22T12:33:06,121 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-07-22T12:33:06,121 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-07-22T12:33:06,122 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-07-22T12:33:06,122 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-07-22T12:33:06,123 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-07-22T12:33:06,123 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-07-22T12:33:06,123 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:33:06,124 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:33:06,125 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:33:06,125 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:33:06,125 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:33:06,126 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:33:06,126 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:33:06,168 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1690029179973_ns3_test-16900290656252' hfile list 2023-07-22T12:33:06,258 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029186218/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-07-22T12:33:06,258 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029186218/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-07-22T12:33:07,286 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-07-22T12:33:07,287 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-07-22T12:33:07,291 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1690029179973_ns3_test-16900290656252 2023-07-22T12:33:07,291 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1690029179973_ns3_test-16900290656252 finished. 2023-07-22T12:33:07,291 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(1014): ns2:test-16900290656251 2023-07-22T12:33:07,291 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(1014): ns3:test-16900290656252 2023-07-22T12:33:07,297 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029171531 2023-07-22T12:33:07,297 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-07-22T12:33:07,703 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:34745/backupUT/backup_1690029171531/.backup.manifest 2023-07-22T12:33:07,703 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029171531 2023-07-22T12:33:07,703 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-07-22T12:33:07,709 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:34745/backupUT/backup_1690029171531/.backup.manifest 2023-07-22T12:33:07,709 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(358): Backup backup_1690029171531 finished: type=FULL,tablelist=ns2:test-16900290656251;ns3:test-16900290656252,targetRootDir=hdfs://localhost:34745/backupUT,startts=1690029175673,completets=1690029187297,bytescopied=0 2023-07-22T12:33:07,709 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-07-22T12:33:07,709 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1690029175856_ns2_test-16900290656251 2023-07-22T12:33:07,710 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1690029175856_ns2_test-16900290656251" 2023-07-22T12:33:07,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1690029175856_ns2_test-16900290656251 2023-07-22T12:33:07,713 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1690029175856_ns2_test-16900290656251 for backup backup_1690029171531 succeeded. 2023-07-22T12:33:07,713 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1690029179973_ns3_test-16900290656252 2023-07-22T12:33:07,714 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1690029179973_ns3_test-16900290656252" 2023-07-22T12:33:07,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1690029179973_ns3_test-16900290656252 2023-07-22T12:33:07,717 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1690029179973_ns3_test-16900290656252 for backup backup_1690029171531 succeeded. 2023-07-22T12:33:07,717 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-07-22T12:33:07,725 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-07-22T12:33:07,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-07-22T12:33:07,728 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-07-22T12:33:07,729 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-07-22T12:33:07,735 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(376): Backup backup_1690029171531 completed. 2023-07-22T12:33:07,736 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(509): Stopping rpc client You have to specify either set name or table list to restore Usage: hbase restore [options] backup_path Path to a backup destination root backup_id Backup image ID to restore table(s) Comma-separated list of tables to restore Options: -c Check restore sequence and dependencies only (does not execute the command) -d Enable debug loggings -h,--help Show usage -m A comma separated list of target tables. If specified, each table in must have a mapping -o Overwrite data if any of the restore target tables exists -q Yarn queue name to run backup restore command on -s Backup set to restore, mutually exclusive with -t (table list) -t Table name list, comma-separated. To enable backup, in hbase-site.xml, set: hbase.backup.enable=true hbase.master.logcleaner.plugins=YOUR_PLUGINS,org.apache.hadoop.hbase.backup.master.BackupLogCleaner hbase.procedure.master.classes=YOUR_CLASSES,org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager hbase.procedure.regionserver.classes=YOUR_CLASSES,org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager hbase.coprocessor.region.classes=YOUR_CLASSES,org.apache.hadoop.hbase.backup.BackupObserver and restart the cluster For more information please see http://hbase.apache.org/book.html#backuprestore 2023-07-22T12:33:07,767 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreMultipleDNECommand Thread=597 (was 604), OpenFileDescriptor=682 (was 691), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=112 (was 124), ProcessCount=169 (was 169), AvailableMemoryMB=16107 (was 16235) 2023-07-22T12:33:07,767 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(130): Thread=597 is superior to 500 2023-07-22T12:33:07,783 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreMultiple Thread=597, OpenFileDescriptor=682, MaxFileDescriptor=60000, SystemLoadAverage=112, ProcessCount=169, AvailableMemoryMB=16107 2023-07-22T12:33:07,783 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(130): Thread=597 is superior to 500 2023-07-22T12:33:07,783 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] backup.TestFullRestore(120): create full backup image on multiple tables 2023-07-22T12:33:07,792 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34e45123, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:33:07,799 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:33:07,800 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:41718, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:33:07,802 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:33:07,804 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:47706, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:33:07,813 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-07-22T12:33:07,818 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:33:07,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029187818 (current time:1690029187818). 2023-07-22T12:33:07,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:33:07,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-07-22T12:33:07,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40df8953, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:33:07,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:33:07,825 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:41730, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:33:07,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:33:07,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:33:07,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=95, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:33:07,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 95 2023-07-22T12:33:07,828 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:33:07,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=95 2023-07-22T12:33:07,829 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:33:07,831 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:33:07,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=95 2023-07-22T12:33:08,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=95 2023-07-22T12:33:08,237 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:33:08,237 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33}] 2023-07-22T12:33:08,239 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=96, ppid=95, state=RUNNABLE; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33 2023-07-22T12:33:08,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43639 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=96 2023-07-22T12:33:08,390 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. 2023-07-22T12:33:08,390 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.HRegion(2819): Flushing 697cc5ecf0962f5ab9f4126febf7ce33 2/2 column families, dataSize=1.50 KB heapSize=2.73 KB 2023-07-22T12:33:08,394 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/meta/b30f291cd15a4d92ac4b31f477e93428 is 171, key is trslm:hdfs://localhost:34745/backupUT\x00ns2:test-16900290656251/meta:log-roll-map/1690029187294/Put/seqid=0 2023-07-22T12:33:08,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=95 2023-07-22T12:33:08,798 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=717 B at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/meta/b30f291cd15a4d92ac4b31f477e93428 2023-07-22T12:33:08,803 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: b30f291cd15a4d92ac4b31f477e93428 2023-07-22T12:33:08,805 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/session/a5d57e73d32d4eda8e52490df82775fa is 524, key is session:backup_1690029171531/session:context/1690029187729/Put/seqid=0 2023-07-22T12:33:08,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=95 2023-07-22T12:33:09,210 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=820 B at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/session/a5d57e73d32d4eda8e52490df82775fa 2023-07-22T12:33:09,217 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a5d57e73d32d4eda8e52490df82775fa 2023-07-22T12:33:09,218 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/meta/b30f291cd15a4d92ac4b31f477e93428 as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/b30f291cd15a4d92ac4b31f477e93428 2023-07-22T12:33:09,223 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/b30f291cd15a4d92ac4b31f477e93428, entries=6, sequenceid=83, filesize=5.8 K 2023-07-22T12:33:09,224 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/session/a5d57e73d32d4eda8e52490df82775fa as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/a5d57e73d32d4eda8e52490df82775fa 2023-07-22T12:33:09,229 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/a5d57e73d32d4eda8e52490df82775fa, entries=2, sequenceid=83, filesize=5.5 K 2023-07-22T12:33:09,229 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.HRegion(3022): Finished flush of dataSize ~1.50 KB/1537, heapSize ~2.70 KB/2760, currentSize=0 B/0 for 697cc5ecf0962f5ab9f4126febf7ce33 in 839ms, sequenceid=83, compaction requested=true 2023-07-22T12:33:09,230 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.HRegion(2520): Flush status journal for 697cc5ecf0962f5ab9f4126febf7ce33: 2023-07-22T12:33:09,230 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. for snapshot_backup_system completed. 2023-07-22T12:33:09,230 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33.' region-info for snapshot=snapshot_backup_system 2023-07-22T12:33:09,230 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:33:09,230 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/b580ac8bece04b07a21605e555bdc696, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/5ea0158e2d574f758c68ee9c20c21191, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/a6d2695c5574440ea495f58a7c66ed03, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/8f78c6cced064299a30d72a0b4901528, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/606f117b1e384a54bb6dba5339e90e8f, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/15bf4f9b1b65467c96713e751468dc6c, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/b30f291cd15a4d92ac4b31f477e93428] hfiles 2023-07-22T12:33:09,230 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (1/7): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/b580ac8bece04b07a21605e555bdc696 for snapshot=snapshot_backup_system 2023-07-22T12:33:09,230 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (2/7): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/5ea0158e2d574f758c68ee9c20c21191 for snapshot=snapshot_backup_system 2023-07-22T12:33:09,230 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (3/7): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/a6d2695c5574440ea495f58a7c66ed03 for snapshot=snapshot_backup_system 2023-07-22T12:33:09,230 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (4/7): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/8f78c6cced064299a30d72a0b4901528 for snapshot=snapshot_backup_system 2023-07-22T12:33:09,230 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (5/7): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/606f117b1e384a54bb6dba5339e90e8f for snapshot=snapshot_backup_system 2023-07-22T12:33:09,230 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (6/7): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/15bf4f9b1b65467c96713e751468dc6c for snapshot=snapshot_backup_system 2023-07-22T12:33:09,230 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (7/7): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/b30f291cd15a4d92ac4b31f477e93428 for snapshot=snapshot_backup_system 2023-07-22T12:33:09,230 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/956010b3eaa64b4fa82ddea4557fac2c, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/0704967b71ce4a5cb0af553e6e47addb, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/62b5d2f1465b492ab802e26f5bf32f6b, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/c2fdeacbf4184719a11904adfeaf1fd8, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/660347589bb3465a82d9c9138151d6a4, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/2b4c40a0167b43f584d34520d18e9390, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/1550986ca81f443894a286cada73692c, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/a5d57e73d32d4eda8e52490df82775fa] hfiles 2023-07-22T12:33:09,230 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (1/8): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/956010b3eaa64b4fa82ddea4557fac2c for snapshot=snapshot_backup_system 2023-07-22T12:33:09,231 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (2/8): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/0704967b71ce4a5cb0af553e6e47addb for snapshot=snapshot_backup_system 2023-07-22T12:33:09,231 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (3/8): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/62b5d2f1465b492ab802e26f5bf32f6b for snapshot=snapshot_backup_system 2023-07-22T12:33:09,231 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (4/8): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/c2fdeacbf4184719a11904adfeaf1fd8 for snapshot=snapshot_backup_system 2023-07-22T12:33:09,231 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (5/8): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/660347589bb3465a82d9c9138151d6a4 for snapshot=snapshot_backup_system 2023-07-22T12:33:09,231 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (6/8): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/2b4c40a0167b43f584d34520d18e9390 for snapshot=snapshot_backup_system 2023-07-22T12:33:09,231 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (7/8): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/1550986ca81f443894a286cada73692c for snapshot=snapshot_backup_system 2023-07-22T12:33:09,231 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (8/8): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/a5d57e73d32d4eda8e52490df82775fa for snapshot=snapshot_backup_system 2023-07-22T12:33:09,637 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. 2023-07-22T12:33:09,637 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=96 2023-07-22T12:33:09,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster(3994): Remote procedure done, pid=96 2023-07-22T12:33:09,637 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 697cc5ecf0962f5ab9f4126febf7ce33 2023-07-22T12:33:09,640 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=96, ppid=95, state=RUNNABLE; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33 2023-07-22T12:33:09,642 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=96, resume processing ppid=95 2023-07-22T12:33:09,642 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=96, ppid=95, state=SUCCESS; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33 in 1.4040 sec 2023-07-22T12:33:09,642 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:33:09,643 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:33:09,643 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:33:09,643 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-07-22T12:33:09,644 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_backup_system 2023-07-22T12:33:09,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=95 2023-07-22T12:33:10,055 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:33:10,066 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:33:10,066 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_backup_system 2023-07-22T12:33:10,883 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:33:10,883 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 95 2023-07-22T12:33:10,885 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=95, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 3.0570 sec 2023-07-22T12:33:11,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=95 2023-07-22T12:33:11,934 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(115): Backup backup_1690029187792 started at 1690029191934. 2023-07-22T12:33:11,936 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(119): Backup session backup_1690029187792 has been started. 2023-07-22T12:33:11,940 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-07-22T12:33:11,941 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-07-22T12:33:11,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-07-22T12:33:11,941 INFO [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-07-22T12:33:11,941 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-07-22T12:33:11,942 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-07-22T12:33:11,942 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-07-22T12:33:11,943 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:33:11,950 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:33:11,950 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:33:11,950 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:33:11,951 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-07-22T12:33:11,951 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-07-22T12:33:11,951 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-07-22T12:33:11,951 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:33:11,951 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-07-22T12:33:11,951 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-07-22T12:33:11,951 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-07-22T12:33:11,952 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-07-22T12:33:11,952 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-07-22T12:33:11,952 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-07-22T12:33:11,952 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-07-22T12:33:11,952 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,43639,1690029056327' joining acquired barrier for procedure (rolllog) in zk 2023-07-22T12:33:11,954 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-07-22T12:33:11,955 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:33:11,955 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:33:11,955 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:33:11,955 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-07-22T12:33:11,955 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:33:11,955 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:33:11,955 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:33:11,956 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:33:11,956 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:33:11,956 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,43639,1690029056327' joining acquired barrier for procedure 'rolllog' on coordinator 2023-07-22T12:33:11,956 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@ce33f04[Count = 0] remaining members to acquire global barrier 2023-07-22T12:33:11,956 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-07-22T12:33:11,956 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-07-22T12:33:11,957 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-07-22T12:33:11,957 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-07-22T12:33:11,958 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-07-22T12:33:11,958 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-07-22T12:33:11,958 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-07-22T12:33:11,958 DEBUG [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:33:11,958 INFO [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1690029175708 highest: 1690029175708 on jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:33:11,958 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta:.meta(num 1690029175691) roll requested 2023-07-22T12:33:11,969 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029191958.meta, exclude list is [], retry=0 2023-07-22T12:33:11,974 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK] 2023-07-22T12:33:11,975 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029175691.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029191958.meta 2023-07-22T12:33:11,975 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK]] 2023-07-22T12:33:11,975 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029175691.meta is not closed yet, will try archiving it next time 2023-07-22T12:33:11,975 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C43639%2C1690029056327:(num 1690029175708) roll requested 2023-07-22T12:33:11,978 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029175691.meta to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/oldWALs/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029175691.meta 2023-07-22T12:33:11,987 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029191975, exclude list is [], retry=0 2023-07-22T12:33:11,991 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK] 2023-07-22T12:33:11,992 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029175708 with entries=10, filesize=2.67 KB; new WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029191975 2023-07-22T12:33:11,993 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK]] 2023-07-22T12:33:11,993 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029175708 is not closed yet, will try archiving it next time 2023-07-22T12:33:11,998 DEBUG [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-07-22T12:33:11,998 INFO [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1690029191975 on jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:33:12,010 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-07-22T12:33:12,010 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,43639,1690029056327' in zk 2023-07-22T12:33:12,013 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-07-22T12:33:12,013 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:33:12,013 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:33:12,013 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:33:12,013 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:33:12,013 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-07-22T12:33:12,014 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:33:12,015 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:33:12,015 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:33:12,016 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:33:12,016 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:33:12,016 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:33:12,017 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:33:12,017 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,43639,1690029056327': 2023-07-22T12:33:12,017 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,43639,1690029056327' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-07-22T12:33:12,017 INFO [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-07-22T12:33:12,017 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-07-22T12:33:12,017 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-07-22T12:33:12,018 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-07-22T12:33:12,018 INFO [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-07-22T12:33:12,021 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-07-22T12:33:12,021 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-07-22T12:33:12,021 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:33:12,021 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:33:12,021 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:33:12,022 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:33:12,022 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:33:12,022 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:33:12,022 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:33:12,022 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:33:12,022 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:33:12,023 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:33:12,023 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:33:12,024 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:33:12,024 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:33:12,024 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:33:12,029 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:33:12,029 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:33:12,029 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:33:12,029 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-07-22T12:33:12,029 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-07-22T12:33:12,031 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:33:12,032 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:33:12,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-07-22T12:33:12,135 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1690029192134_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } 2023-07-22T12:33:12,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029192135 (current time:1690029192135). 2023-07-22T12:33:12,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:33:12,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1690029192134_ns2_test-16900290656251 VERSION not specified, setting to 2 2023-07-22T12:33:12,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@510dd9b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:33:12,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:33:12,144 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:59000, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:33:12,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:33:12,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:33:12,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=97, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1690029192134_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } 2023-07-22T12:33:12,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1690029192134_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 }, snapshot procedure id = 97 2023-07-22T12:33:12,147 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1690029192134_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:33:12,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=97 2023-07-22T12:33:12,148 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1690029192134_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:33:12,150 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1690029192134_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:33:12,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=97 2023-07-22T12:33:12,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=97 2023-07-22T12:33:12,555 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1690029192134_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:33:12,555 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE; SnapshotRegionProcedure b8196e3466217e5359c3acacaaeb9e61}] 2023-07-22T12:33:12,557 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=98, ppid=97, state=RUNNABLE; SnapshotRegionProcedure b8196e3466217e5359c3acacaaeb9e61 2023-07-22T12:33:12,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43639 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=98 2023-07-22T12:33:12,709 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-16900290656251,,1690029096058.b8196e3466217e5359c3acacaaeb9e61. 2023-07-22T12:33:12,709 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] regionserver.HRegion(2520): Flush status journal for b8196e3466217e5359c3acacaaeb9e61: 2023-07-22T12:33:12,709 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-16900290656251,,1690029096058.b8196e3466217e5359c3acacaaeb9e61. for snapshot_1690029192134_ns2_test-16900290656251 completed. 2023-07-22T12:33:12,709 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] snapshot.SnapshotManifest(240): Storing 'ns2:test-16900290656251,,1690029096058.b8196e3466217e5359c3acacaaeb9e61.' region-info for snapshot=snapshot_1690029192134_ns2_test-16900290656251 2023-07-22T12:33:12,709 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:33:12,709 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/b8196e3466217e5359c3acacaaeb9e61/f/82851155b50f47edb054b7e8d9f06090_SeqId_4_] hfiles 2023-07-22T12:33:12,709 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/b8196e3466217e5359c3acacaaeb9e61/f/82851155b50f47edb054b7e8d9f06090_SeqId_4_ for snapshot=snapshot_1690029192134_ns2_test-16900290656251 2023-07-22T12:33:12,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=97 2023-07-22T12:33:13,116 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-16900290656251,,1690029096058.b8196e3466217e5359c3acacaaeb9e61. 2023-07-22T12:33:13,116 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=98 2023-07-22T12:33:13,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster(3994): Remote procedure done, pid=98 2023-07-22T12:33:13,117 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1690029192134_ns2_test-16900290656251 on region b8196e3466217e5359c3acacaaeb9e61 2023-07-22T12:33:13,118 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=98, ppid=97, state=RUNNABLE; SnapshotRegionProcedure b8196e3466217e5359c3acacaaeb9e61 2023-07-22T12:33:13,120 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=98, resume processing ppid=97 2023-07-22T12:33:13,120 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1690029192134_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:33:13,120 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=98, ppid=97, state=SUCCESS; SnapshotRegionProcedure b8196e3466217e5359c3acacaaeb9e61 in 564 msec 2023-07-22T12:33:13,121 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1690029192134_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:33:13,121 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1690029192134_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:33:13,121 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1690029192134_ns2_test-16900290656251 2023-07-22T12:33:13,123 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_1690029192134_ns2_test-16900290656251 2023-07-22T12:33:13,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=97 2023-07-22T12:33:13,533 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1690029192134_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:33:13,539 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1690029192134_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:33:13,540 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_1690029192134_ns2_test-16900290656251 to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029192134_ns2_test-16900290656251 2023-07-22T12:33:14,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=97 2023-07-22T12:33:14,356 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1690029192134_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:33:14,356 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1690029192134_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 }, snapshot procedure id = 97 2023-07-22T12:33:14,357 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=97, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1690029192134_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } in 2.2110 sec 2023-07-22T12:33:16,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=97 2023-07-22T12:33:16,254 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1690029196254_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } 2023-07-22T12:33:16,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029196254 (current time:1690029196254). 2023-07-22T12:33:16,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:33:16,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1690029196254_ns3_test-16900290656252 VERSION not specified, setting to 2 2023-07-22T12:33:16,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70a61a00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:33:16,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:33:16,263 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:59004, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:33:16,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:33:16,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:33:16,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=99, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1690029196254_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } 2023-07-22T12:33:16,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1690029196254_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 }, snapshot procedure id = 99 2023-07-22T12:33:16,267 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1690029196254_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:33:16,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=99 2023-07-22T12:33:16,267 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1690029196254_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:33:16,269 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1690029196254_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:33:16,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=99 2023-07-22T12:33:16,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=99 2023-07-22T12:33:16,674 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1690029196254_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:33:16,674 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE; SnapshotRegionProcedure aefea80783ea312734eb5d79ad00b2f1}] 2023-07-22T12:33:16,676 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=100, ppid=99, state=RUNNABLE; SnapshotRegionProcedure aefea80783ea312734eb5d79ad00b2f1 2023-07-22T12:33:16,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43639 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=100 2023-07-22T12:33:16,828 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=100}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-16900290656252,,1690029100393.aefea80783ea312734eb5d79ad00b2f1. 2023-07-22T12:33:16,828 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=100}] regionserver.HRegion(2520): Flush status journal for aefea80783ea312734eb5d79ad00b2f1: 2023-07-22T12:33:16,828 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=100}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-16900290656252,,1690029100393.aefea80783ea312734eb5d79ad00b2f1. for snapshot_1690029196254_ns3_test-16900290656252 completed. 2023-07-22T12:33:16,828 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=100}] snapshot.SnapshotManifest(240): Storing 'ns3:test-16900290656252,,1690029100393.aefea80783ea312734eb5d79ad00b2f1.' region-info for snapshot=snapshot_1690029196254_ns3_test-16900290656252 2023-07-22T12:33:16,828 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=100}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:33:16,828 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=100}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-07-22T12:33:16,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=99 2023-07-22T12:33:17,235 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=100}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-16900290656252,,1690029100393.aefea80783ea312734eb5d79ad00b2f1. 2023-07-22T12:33:17,235 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=100 2023-07-22T12:33:17,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster(3994): Remote procedure done, pid=100 2023-07-22T12:33:17,235 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1690029196254_ns3_test-16900290656252 on region aefea80783ea312734eb5d79ad00b2f1 2023-07-22T12:33:17,237 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=100, ppid=99, state=RUNNABLE; SnapshotRegionProcedure aefea80783ea312734eb5d79ad00b2f1 2023-07-22T12:33:17,239 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=100, resume processing ppid=99 2023-07-22T12:33:17,239 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=100, ppid=99, state=SUCCESS; SnapshotRegionProcedure aefea80783ea312734eb5d79ad00b2f1 in 564 msec 2023-07-22T12:33:17,239 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1690029196254_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:33:17,240 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1690029196254_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:33:17,240 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1690029196254_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:33:17,240 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1690029196254_ns3_test-16900290656252 2023-07-22T12:33:17,241 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_1690029196254_ns3_test-16900290656252 2023-07-22T12:33:17,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=99 2023-07-22T12:33:17,650 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1690029196254_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:33:17,654 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1690029196254_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:33:17,655 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_1690029196254_ns3_test-16900290656252 to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029196254_ns3_test-16900290656252 2023-07-22T12:33:18,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=99 2023-07-22T12:33:18,471 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1690029196254_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:33:18,471 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1690029196254_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 }, snapshot procedure id = 99 2023-07-22T12:33:18,471 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=99, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1690029196254_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } in 2.2060 sec 2023-07-22T12:33:20,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=99 2023-07-22T12:33:20,373 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1690029187792 2023-07-22T12:33:20,373 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-07-22T12:33:20,373 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2023-07-22T12:33:20,373 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1690029187792_ns2:test-16900290656251 2023-07-22T12:33:20,373 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1690029192134_ns2_test-16900290656251 to hdfs://localhost:34745/backupUT/backup_1690029187792/ns2/test-16900290656251/ 2023-07-22T12:33:20,373 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-07-22T12:33:20,396 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:34745, inputRoot=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947 2023-07-22T12:33:20,396 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,37923,1690029053050_948887521_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:34745/backupUT/backup_1690029187792/ns2/test-16900290656251, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:34745/backupUT/backup_1690029187792/ns2/test-16900290656251/.hbase-snapshot/.tmp/snapshot_1690029192134_ns2_test-16900290656251 2023-07-22T12:33:20,396 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:34745, inputRoot=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947, snapshotDir=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029192134_ns2_test-16900290656251. 2023-07-22T12:33:20,405 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029192134_ns2_test-16900290656251 to hdfs://localhost:34745/backupUT/backup_1690029187792/ns2/test-16900290656251/.hbase-snapshot/.tmp/snapshot_1690029192134_ns2_test-16900290656251 2023-07-22T12:33:20,826 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:20,826 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:20,826 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:20,827 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:21,797 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/hadoop-1996900908712431344.jar 2023-07-22T12:33:21,797 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:21,797 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:21,798 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:21,798 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:21,799 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:21,799 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:21,800 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:21,800 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:21,800 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:21,801 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:33:21,801 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-07-22T12:33:21,802 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-07-22T12:33:21,802 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-07-22T12:33:21,803 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-07-22T12:33:21,803 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-07-22T12:33:21,803 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-07-22T12:33:21,804 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-07-22T12:33:21,804 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-07-22T12:33:21,804 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-07-22T12:33:21,805 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-07-22T12:33:21,805 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-07-22T12:33:21,806 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:33:21,806 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:33:21,807 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:33:21,807 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:33:21,807 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:33:21,808 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:33:21,808 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:33:21,849 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1690029192134_ns2_test-16900290656251' hfile list 2023-07-22T12:33:21,852 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-07-22T12:33:21,922 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029201889/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-07-22T12:33:21,922 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029201889/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-07-22T12:33:21,973 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-07-22T12:33:21,997 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/b8196e3466217e5359c3acacaaeb9e61/f/82851155b50f47edb054b7e8d9f06090_SeqId_4_ output=hdfs://localhost:34745/backupUT/backup_1690029187792/ns2/test-16900290656251/archive/data/ns2/test-16900290656251/b8196e3466217e5359c3acacaaeb9e61/f/82851155b50f47edb054b7e8d9f06090_SeqId_4_ 2023-07-22T12:33:21,997 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8725 (8.5 K) time=0sec 8.321M/sec 2023-07-22T12:33:22,956 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-07-22T12:33:22,957 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-07-22T12:33:22,963 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1690029192134_ns2_test-16900290656251 2023-07-22T12:33:22,963 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1690029192134_ns2_test-16900290656251 finished. 2023-07-22T12:33:22,963 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1690029187792_ns3:test-16900290656252 2023-07-22T12:33:22,963 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1690029196254_ns3_test-16900290656252 to hdfs://localhost:34745/backupUT/backup_1690029187792/ns3/test-16900290656252/ 2023-07-22T12:33:22,963 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-07-22T12:33:22,986 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:34745, inputRoot=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947 2023-07-22T12:33:22,986 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,37923,1690029053050_842596486_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:34745/backupUT/backup_1690029187792/ns3/test-16900290656252, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:34745/backupUT/backup_1690029187792/ns3/test-16900290656252/.hbase-snapshot/.tmp/snapshot_1690029196254_ns3_test-16900290656252 2023-07-22T12:33:22,986 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:34745, inputRoot=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947, snapshotDir=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029196254_ns3_test-16900290656252. 2023-07-22T12:33:22,990 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029196254_ns3_test-16900290656252 to hdfs://localhost:34745/backupUT/backup_1690029187792/ns3/test-16900290656252/.hbase-snapshot/.tmp/snapshot_1690029196254_ns3_test-16900290656252 2023-07-22T12:33:23,407 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:23,408 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:23,408 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:23,409 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:24,412 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/hadoop-7494274100600395261.jar 2023-07-22T12:33:24,412 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:24,413 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:24,413 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:24,414 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:24,414 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:24,414 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:24,415 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:24,415 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:24,415 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:24,416 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:33:24,416 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-07-22T12:33:24,417 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-07-22T12:33:24,417 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-07-22T12:33:24,418 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-07-22T12:33:24,418 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-07-22T12:33:24,418 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-07-22T12:33:24,419 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-07-22T12:33:24,419 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-07-22T12:33:24,419 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-07-22T12:33:24,420 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-07-22T12:33:24,420 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-07-22T12:33:24,421 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:33:24,421 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:33:24,422 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:33:24,422 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:33:24,422 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:33:24,423 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:33:24,423 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:33:24,466 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1690029196254_ns3_test-16900290656252' hfile list 2023-07-22T12:33:24,534 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029204503/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-07-22T12:33:24,534 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029204503/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-07-22T12:33:25,583 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-07-22T12:33:25,585 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-07-22T12:33:25,591 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1690029196254_ns3_test-16900290656252 2023-07-22T12:33:25,591 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1690029196254_ns3_test-16900290656252 finished. 2023-07-22T12:33:25,592 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(1014): ns2:test-16900290656251 2023-07-22T12:33:25,592 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(1014): ns3:test-16900290656252 2023-07-22T12:33:25,598 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029187792 2023-07-22T12:33:25,598 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-07-22T12:33:25,605 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:34745/backupUT/backup_1690029187792/.backup.manifest 2023-07-22T12:33:25,605 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029187792 2023-07-22T12:33:25,605 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-07-22T12:33:26,010 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:34745/backupUT/backup_1690029187792/.backup.manifest 2023-07-22T12:33:26,010 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(358): Backup backup_1690029187792 finished: type=FULL,tablelist=ns2:test-16900290656251;ns3:test-16900290656252,targetRootDir=hdfs://localhost:34745/backupUT,startts=1690029191934,completets=1690029205598,bytescopied=0 2023-07-22T12:33:26,010 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-07-22T12:33:26,010 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1690029192134_ns2_test-16900290656251 2023-07-22T12:33:26,011 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1690029192134_ns2_test-16900290656251" 2023-07-22T12:33:26,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1690029192134_ns2_test-16900290656251 2023-07-22T12:33:26,014 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1690029192134_ns2_test-16900290656251 for backup backup_1690029187792 succeeded. 2023-07-22T12:33:26,014 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1690029196254_ns3_test-16900290656252 2023-07-22T12:33:26,015 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1690029196254_ns3_test-16900290656252" 2023-07-22T12:33:26,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1690029196254_ns3_test-16900290656252 2023-07-22T12:33:26,017 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1690029196254_ns3_test-16900290656252 for backup backup_1690029187792 succeeded. 2023-07-22T12:33:26,018 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-07-22T12:33:26,022 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-07-22T12:33:26,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-07-22T12:33:26,024 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-07-22T12:33:26,026 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-07-22T12:33:26,031 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(376): Backup backup_1690029187792 completed. 2023-07-22T12:33:26,032 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:33:26,044 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(101): HBase table ns2:table2 does not exist. It will be created during restore process 2023-07-22T12:33:26,045 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(101): HBase table ns3:table3_restore does not exist. It will be created during restore process 2023-07-22T12:33:26,046 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34745/backupUT/backup_1690029187792/.backup.manifest 2023-07-22T12:33:26,048 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029187792/.backup.manifest 2023-07-22T12:33:26,048 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34745/backupUT/backup_1690029187792/.backup.manifest 2023-07-22T12:33:26,049 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029187792/.backup.manifest 2023-07-22T12:33:26,050 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34745/backupUT/backup_1690029187792/.backup.manifest 2023-07-22T12:33:26,051 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029187792/.backup.manifest 2023-07-22T12:33:26,052 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(148): Restoring 'ns2:test-16900290656251' to 'ns2:table2' from full backup image hdfs://localhost:34745/backupUT/backup_1690029187792/ns2/test-16900290656251 2023-07-22T12:33:26,057 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] util.RestoreTool(487): Creating target table 'ns2:table2' 2023-07-22T12:33:26,057 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] util.RestoreTool(391): Parsing region dir: hdfs://localhost:34745/backupUT/backup_1690029187792/ns2/test-16900290656251/archive/data/ns2/test-16900290656251/b8196e3466217e5359c3acacaaeb9e61 2023-07-22T12:33:26,058 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:34745/backupUT/backup_1690029187792/ns2/test-16900290656251/archive/data/ns2/test-16900290656251/b8196e3466217e5359c3acacaaeb9e61/f in region [hdfs://localhost:34745/backupUT/backup_1690029187792/ns2/test-16900290656251/archive/data/ns2/test-16900290656251/b8196e3466217e5359c3acacaaeb9e61] 2023-07-22T12:33:26,061 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] util.RestoreTool(442): Trying to figure out region boundaries hfile=hdfs://localhost:34745/backupUT/backup_1690029187792/ns2/test-16900290656251/archive/data/ns2/test-16900290656251/b8196e3466217e5359c3acacaaeb9e61/f/82851155b50f47edb054b7e8d9f06090_SeqId_4_ first=row0 last=row98 2023-07-22T12:33:26,061 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 82851155b50f47edb054b7e8d9f06090_SeqId_4_ 2023-07-22T12:33:26,062 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'ns2:table2', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:33:26,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=101, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns2:table2 2023-07-22T12:33:26,064 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=101, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:33:26,064 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns2" qualifier: "table2" procId is: 101 2023-07-22T12:33:26,065 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=101, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:33:26,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=101 2023-07-22T12:33:26,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=101 2023-07-22T12:33:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=101 2023-07-22T12:33:26,472 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 78b093193ab7ebb2bb1fe03dad7dcd4a, NAME => 'ns2:table2,,1690029206061.78b093193ab7ebb2bb1fe03dad7dcd4a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:table2', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947 2023-07-22T12:33:26,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=101 2023-07-22T12:33:26,877 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:table2,,1690029206061.78b093193ab7ebb2bb1fe03dad7dcd4a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:33:26,877 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1668): Closing 78b093193ab7ebb2bb1fe03dad7dcd4a, disabling compactions & flushes 2023-07-22T12:33:26,878 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:table2,,1690029206061.78b093193ab7ebb2bb1fe03dad7dcd4a. 2023-07-22T12:33:26,878 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:table2,,1690029206061.78b093193ab7ebb2bb1fe03dad7dcd4a. 2023-07-22T12:33:26,878 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:table2,,1690029206061.78b093193ab7ebb2bb1fe03dad7dcd4a. after waiting 0 ms 2023-07-22T12:33:26,878 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:table2,,1690029206061.78b093193ab7ebb2bb1fe03dad7dcd4a. 2023-07-22T12:33:26,878 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1909): Closed ns2:table2,,1690029206061.78b093193ab7ebb2bb1fe03dad7dcd4a. 2023-07-22T12:33:26,878 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1622): Region close journal for 78b093193ab7ebb2bb1fe03dad7dcd4a: 2023-07-22T12:33:26,879 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=101, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:33:26,879 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:table2,,1690029206061.78b093193ab7ebb2bb1fe03dad7dcd4a.","families":{"info":[{"qualifier":"regioninfo","vlen":36,"tag":[],"timestamp":"1690029206879"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690029206879"}]},"ts":"1690029206879"} 2023-07-22T12:33:26,881 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:33:26,882 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=101, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:33:26,882 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029206882"}]},"ts":"1690029206882"} 2023-07-22T12:33:26,883 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:table2, state=ENABLING in hbase:meta 2023-07-22T12:33:26,887 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:table2, region=78b093193ab7ebb2bb1fe03dad7dcd4a, ASSIGN}] 2023-07-22T12:33:26,888 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:table2, region=78b093193ab7ebb2bb1fe03dad7dcd4a, ASSIGN 2023-07-22T12:33:26,889 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:table2, region=78b093193ab7ebb2bb1fe03dad7dcd4a, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,43639,1690029056327; forceNewPlan=false, retain=false 2023-07-22T12:33:27,040 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=102 updating hbase:meta row=78b093193ab7ebb2bb1fe03dad7dcd4a, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:33:27,042 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=103, ppid=102, state=RUNNABLE; OpenRegionProcedure 78b093193ab7ebb2bb1fe03dad7dcd4a, server=jenkins-hbase3.apache.org,43639,1690029056327}] 2023-07-22T12:33:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=101 2023-07-22T12:33:27,196 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] handler.AssignRegionHandler(130): Open ns2:table2,,1690029206061.78b093193ab7ebb2bb1fe03dad7dcd4a. 2023-07-22T12:33:27,196 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(7254): Opening region: {ENCODED => 78b093193ab7ebb2bb1fe03dad7dcd4a, NAME => 'ns2:table2,,1690029206061.78b093193ab7ebb2bb1fe03dad7dcd4a.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:33:27,197 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:33:27,197 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table2 78b093193ab7ebb2bb1fe03dad7dcd4a 2023-07-22T12:33:27,197 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(888): Instantiated ns2:table2,,1690029206061.78b093193ab7ebb2bb1fe03dad7dcd4a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:33:27,197 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(7296): checking encryption for 78b093193ab7ebb2bb1fe03dad7dcd4a 2023-07-22T12:33:27,197 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(7299): checking classloading for 78b093193ab7ebb2bb1fe03dad7dcd4a 2023-07-22T12:33:27,199 INFO [StoreOpener-78b093193ab7ebb2bb1fe03dad7dcd4a-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 78b093193ab7ebb2bb1fe03dad7dcd4a 2023-07-22T12:33:27,200 INFO [StoreOpener-78b093193ab7ebb2bb1fe03dad7dcd4a-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 78b093193ab7ebb2bb1fe03dad7dcd4a columnFamilyName f 2023-07-22T12:33:27,200 DEBUG [StoreOpener-78b093193ab7ebb2bb1fe03dad7dcd4a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:33:27,201 INFO [StoreOpener-78b093193ab7ebb2bb1fe03dad7dcd4a-1 {}] regionserver.HStore(324): Store=78b093193ab7ebb2bb1fe03dad7dcd4a/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:33:27,202 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/table2/78b093193ab7ebb2bb1fe03dad7dcd4a 2023-07-22T12:33:27,202 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/table2/78b093193ab7ebb2bb1fe03dad7dcd4a 2023-07-22T12:33:27,205 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(1076): writing seq id for 78b093193ab7ebb2bb1fe03dad7dcd4a 2023-07-22T12:33:27,207 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/table2/78b093193ab7ebb2bb1fe03dad7dcd4a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:33:27,207 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(1093): Opened 78b093193ab7ebb2bb1fe03dad7dcd4a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11599756000, jitterRate=0.08031146228313446}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-07-22T12:33:27,208 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(992): Region open journal for 78b093193ab7ebb2bb1fe03dad7dcd4a: 2023-07-22T12:33:27,209 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegionServer(2552): Post open deploy tasks for ns2:table2,,1690029206061.78b093193ab7ebb2bb1fe03dad7dcd4a., pid=103, masterSystemTime=1690029207193 2023-07-22T12:33:27,210 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegionServer(2579): Finished post open deploy task for ns2:table2,,1690029206061.78b093193ab7ebb2bb1fe03dad7dcd4a. 2023-07-22T12:33:27,210 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] handler.AssignRegionHandler(158): Opened ns2:table2,,1690029206061.78b093193ab7ebb2bb1fe03dad7dcd4a. 2023-07-22T12:33:27,210 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=102 updating hbase:meta row=78b093193ab7ebb2bb1fe03dad7dcd4a, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:33:27,213 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=103, resume processing ppid=102 2023-07-22T12:33:27,213 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=103, ppid=102, state=SUCCESS; OpenRegionProcedure 78b093193ab7ebb2bb1fe03dad7dcd4a, server=jenkins-hbase3.apache.org,43639,1690029056327 in 170 msec 2023-07-22T12:33:27,214 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=102, resume processing ppid=101 2023-07-22T12:33:27,214 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=102, ppid=101, state=SUCCESS; TransitRegionStateProcedure table=ns2:table2, region=78b093193ab7ebb2bb1fe03dad7dcd4a, ASSIGN in 326 msec 2023-07-22T12:33:27,215 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=101, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:33:27,215 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029207215"}]},"ts":"1690029207215"} 2023-07-22T12:33:27,216 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:table2, state=ENABLED in hbase:meta 2023-07-22T12:33:27,219 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=101, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:33:27,220 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=101, state=SUCCESS; CreateTableProcedure table=ns2:table2 in 1.1570 sec 2023-07-22T12:33:28,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=101 2023-07-22T12:33:28,173 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:34745/backupUT/backup_1690029187792/ns2/test-16900290656251/archive/data/ns2/test-16900290656251/b8196e3466217e5359c3acacaaeb9e61 from hbase tables ns2:test-16900290656251 to tables ns2:table2 2023-07-22T12:33:28,173 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceRestoreJob(72): Restore ns2:test-16900290656251 into ns2:table2 2023-07-22T12:33:28,175 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-ns2-table2-1690029208173 from hdfs://localhost:34745/backupUT/backup_1690029187792/ns2/test-16900290656251/archive/data/ns2/test-16900290656251/b8196e3466217e5359c3acacaaeb9e61 2023-07-22T12:33:28,180 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a1682c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:33:28,181 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:33:28,182 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:39028, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:33:28,183 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-07-22T12:33:28,183 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:table2 2023-07-22T12:33:28,186 DEBUG [hconnection-0x39b91e16-shared-pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:33:28,186 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:55412, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:33:28,187 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:table2: 2023-07-22T12:33:28,188 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-07-22T12:33:28,188 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/partitions_89bd72b0-cb37-4c4a-9708-1654d78f405a 2023-07-22T12:33:28,194 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:28,195 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:28,195 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:28,196 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:29,196 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/hadoop-1565786465106729638.jar 2023-07-22T12:33:29,196 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:29,196 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:29,197 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:29,197 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:29,198 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:29,198 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:29,198 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:29,199 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:29,199 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:29,200 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:33:29,200 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-07-22T12:33:29,200 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-07-22T12:33:29,201 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-07-22T12:33:29,201 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-07-22T12:33:29,202 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-07-22T12:33:29,202 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-07-22T12:33:29,202 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-07-22T12:33:29,203 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-07-22T12:33:29,203 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-07-22T12:33:29,204 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-07-22T12:33:29,204 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-07-22T12:33:29,205 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:29,205 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:29,206 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:29,206 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:29,207 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:29,207 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:29,208 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:33:29,209 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: ns2:table2 2023-07-22T12:33:29,209 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:33:29,209 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-07-22T12:33:29,209 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-07-22T12:33:29,210 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:33:29,223 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-07-22T12:33:29,329 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029209290/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-07-22T12:33:29,329 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029209290/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-07-22T12:33:29,389 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:34745/backupUT/backup_1690029187792/ns2/test-16900290656251/archive/data/ns2/test-16900290656251/b8196e3466217e5359c3acacaaeb9e61/f/82851155b50f47edb054b7e8d9f06090_SeqId_4_ 2023-07-22T12:33:29,393 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-07-22T12:33:29,395 DEBUG [LocalJobRunner Map Task Executor #0 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 82851155b50f47edb054b7e8d9f06090_SeqId_4_ 2023-07-22T12:33:29,422 DEBUG [pool-394-thread-1 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ecf2917, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:33:29,423 DEBUG [hconnection-0x4ae3e07-metaLookup-shared--pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:33:29,425 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:55416, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:33:29,426 DEBUG [pool-394-thread-1 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:33:29,428 DEBUG [pool-394-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-07-22T12:33:29,428 DEBUG [pool-394-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-07-22T12:33:29,434 DEBUG [pool-394-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-ns2-table2-1690029208173/_temporary/0/_temporary/attempt_local667374348_0016_r_000000_0/f/d46097076a8b4465b48a5f151f69a892 is 37, key is row10/f:q1/1690029070502/Put/seqid=0 2023-07-22T12:33:29,435 WARN [Thread-1868 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:43639] Specified favored nodes: [jenkins-hbase3.apache.org:43639] 2023-07-22T12:33:29,512 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:table2' 2023-07-22T12:33:30,386 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-ns2-table2-1690029208173 2023-07-22T12:33:30,391 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@290bf67e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:33:30,392 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:33:30,394 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:37426, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:33:30,399 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-ns2-table2-1690029208173/_SUCCESS 2023-07-22T12:33:30,403 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:33:30,404 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:36582, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:33:30,414 DEBUG [LoadIncrementalHFiles-0 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: d46097076a8b4465b48a5f151f69a892 2023-07-22T12:33:30,414 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-ns2-table2-1690029208173/f/d46097076a8b4465b48a5f151f69a892 first=Optional[row0] last=Optional[row98] 2023-07-22T12:33:30,415 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=ns2:table2,,1690029206061.78b093193ab7ebb2bb1fe03dad7dcd4a., hostname=jenkins-hbase3.apache.org,43639,1690029056327, seqNum=2 for row with hfile group [{f,hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-ns2-table2-1690029208173/f/d46097076a8b4465b48a5f151f69a892}] 2023-07-22T12:33:30,422 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-ns2-table2-1690029208173/f/d46097076a8b4465b48a5f151f69a892 for inclusion in 78b093193ab7ebb2bb1fe03dad7dcd4a/f 2023-07-22T12:33:30,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-07-22T12:33:30,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HStore(625): Region bounds: first= last= 2023-07-22T12:33:30,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: d46097076a8b4465b48a5f151f69a892 2023-07-22T12:33:30,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HRegion(2520): Flush status journal for 78b093193ab7ebb2bb1fe03dad7dcd4a: 2023-07-22T12:33:30,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-ns2-table2-1690029208173/f/d46097076a8b4465b48a5f151f69a892 to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/staging/jenkins__ns2_table2__cjeait1hkqvn623mqb25gob1lkqo886fdnsqeasaj49j11j95b4hiptg83ue65l8/f/d46097076a8b4465b48a5f151f69a892 2023-07-22T12:33:30,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/staging/jenkins__ns2_table2__cjeait1hkqvn623mqb25gob1lkqo886fdnsqeasaj49j11j95b4hiptg83ue65l8/f/d46097076a8b4465b48a5f151f69a892 as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/table2/78b093193ab7ebb2bb1fe03dad7dcd4a/f/5bd07ba136024a6f9c01859bff171d6c_SeqId_4_ 2023-07-22T12:33:30,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4961d4b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:33:30,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:33:30,439 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:36598, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:33:30,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:33:30,446 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:37432, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:33:30,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:33:30,466 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/staging/jenkins__ns2_table2__cjeait1hkqvn623mqb25gob1lkqo886fdnsqeasaj49j11j95b4hiptg83ue65l8/f/d46097076a8b4465b48a5f151f69a892 into 78b093193ab7ebb2bb1fe03dad7dcd4a/f as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/table2/78b093193ab7ebb2bb1fe03dad7dcd4a/f/5bd07ba136024a6f9c01859bff171d6c_SeqId_4_ - updating store file list. 2023-07-22T12:33:30,471 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/table2/78b093193ab7ebb2bb1fe03dad7dcd4a/f/5bd07ba136024a6f9c01859bff171d6c_SeqId_4_ into 78b093193ab7ebb2bb1fe03dad7dcd4a/f 2023-07-22T12:33:30,471 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/staging/jenkins__ns2_table2__cjeait1hkqvn623mqb25gob1lkqo886fdnsqeasaj49j11j95b4hiptg83ue65l8/f/d46097076a8b4465b48a5f151f69a892 into 78b093193ab7ebb2bb1fe03dad7dcd4a/f (new location: hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/table2/78b093193ab7ebb2bb1fe03dad7dcd4a/f/5bd07ba136024a6f9c01859bff171d6c_SeqId_4_) 2023-07-22T12:33:30,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/staging/jenkins__ns2_table2__cjeait1hkqvn623mqb25gob1lkqo886fdnsqeasaj49j11j95b4hiptg83ue65l8/f/d46097076a8b4465b48a5f151f69a892 2023-07-22T12:33:30,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@754db5ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:33:30,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:33:30,482 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:36612, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-07-22T12:33:30,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:33:30,488 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:37448, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-07-22T12:33:30,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:33:30,528 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:33:30,528 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-07-22T12:33:30,528 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-07-22T12:33:30,529 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(233): Backup: backup_1690029187792 hdfs://localhost:34745/backupUT/backup_1690029187792/ns2/test-16900290656251/ 2023-07-22T12:33:30,530 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34745/backupUT/backup_1690029187792/.backup.manifest 2023-07-22T12:33:30,540 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029187792/.backup.manifest 2023-07-22T12:33:30,540 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(148): Restoring 'ns3:test-16900290656252' to 'ns3:table3_restore' from full backup image hdfs://localhost:34745/backupUT/backup_1690029187792/ns3/test-16900290656252 2023-07-22T12:33:30,563 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] util.RestoreTool(101): Folder tableArchivePath: hdfs://localhost:34745/backupUT/backup_1690029187792/ns3/test-16900290656252/archive/data/ns3/test-16900290656252 does not exists 2023-07-22T12:33:30,563 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] util.RestoreTool(326): find table descriptor but no archive dir for table ns3:test-16900290656252, will only create table 2023-07-22T12:33:30,565 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] util.RestoreTool(487): Creating target table 'ns3:table3_restore' 2023-07-22T12:33:30,566 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'ns3:table3_restore', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:33:30,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=104, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns3:table3_restore 2023-07-22T12:33:30,570 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=104, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:33:30,570 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns3" qualifier: "table3_restore" procId is: 104 2023-07-22T12:33:30,571 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=104, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:33:30,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=104 2023-07-22T12:33:30,586 INFO [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => cdacde6e0baab5e76b534080f36b0594, NAME => 'ns3:table3_restore,,1690029210566.cdacde6e0baab5e76b534080f36b0594.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:table3_restore', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947 2023-07-22T12:33:30,592 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:table3_restore,,1690029210566.cdacde6e0baab5e76b534080f36b0594.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:33:30,592 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1668): Closing cdacde6e0baab5e76b534080f36b0594, disabling compactions & flushes 2023-07-22T12:33:30,592 INFO [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:table3_restore,,1690029210566.cdacde6e0baab5e76b534080f36b0594. 2023-07-22T12:33:30,592 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:table3_restore,,1690029210566.cdacde6e0baab5e76b534080f36b0594. 2023-07-22T12:33:30,592 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:table3_restore,,1690029210566.cdacde6e0baab5e76b534080f36b0594. after waiting 0 ms 2023-07-22T12:33:30,592 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:table3_restore,,1690029210566.cdacde6e0baab5e76b534080f36b0594. 2023-07-22T12:33:30,592 INFO [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1909): Closed ns3:table3_restore,,1690029210566.cdacde6e0baab5e76b534080f36b0594. 2023-07-22T12:33:30,592 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1622): Region close journal for cdacde6e0baab5e76b534080f36b0594: 2023-07-22T12:33:30,593 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=104, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:33:30,593 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:table3_restore,,1690029210566.cdacde6e0baab5e76b534080f36b0594.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1690029210593"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690029210593"}]},"ts":"1690029210593"} 2023-07-22T12:33:30,595 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:33:30,596 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=104, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:33:30,596 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029210596"}]},"ts":"1690029210596"} 2023-07-22T12:33:30,597 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:table3_restore, state=ENABLING in hbase:meta 2023-07-22T12:33:30,604 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:table3_restore, region=cdacde6e0baab5e76b534080f36b0594, ASSIGN}] 2023-07-22T12:33:30,606 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:table3_restore, region=cdacde6e0baab5e76b534080f36b0594, ASSIGN 2023-07-22T12:33:30,606 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:table3_restore, region=cdacde6e0baab5e76b534080f36b0594, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,43639,1690029056327; forceNewPlan=false, retain=false 2023-07-22T12:33:30,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=104 2023-07-22T12:33:30,758 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=105 updating hbase:meta row=cdacde6e0baab5e76b534080f36b0594, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:33:30,759 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE; OpenRegionProcedure cdacde6e0baab5e76b534080f36b0594, server=jenkins-hbase3.apache.org,43639,1690029056327}] 2023-07-22T12:33:30,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=104 2023-07-22T12:33:30,914 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] handler.AssignRegionHandler(130): Open ns3:table3_restore,,1690029210566.cdacde6e0baab5e76b534080f36b0594. 2023-07-22T12:33:30,914 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7254): Opening region: {ENCODED => cdacde6e0baab5e76b534080f36b0594, NAME => 'ns3:table3_restore,,1690029210566.cdacde6e0baab5e76b534080f36b0594.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:33:30,914 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:33:30,914 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table3_restore cdacde6e0baab5e76b534080f36b0594 2023-07-22T12:33:30,914 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(888): Instantiated ns3:table3_restore,,1690029210566.cdacde6e0baab5e76b534080f36b0594.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:33:30,914 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7296): checking encryption for cdacde6e0baab5e76b534080f36b0594 2023-07-22T12:33:30,914 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7299): checking classloading for cdacde6e0baab5e76b534080f36b0594 2023-07-22T12:33:30,915 INFO [StoreOpener-cdacde6e0baab5e76b534080f36b0594-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region cdacde6e0baab5e76b534080f36b0594 2023-07-22T12:33:30,917 INFO [StoreOpener-cdacde6e0baab5e76b534080f36b0594-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cdacde6e0baab5e76b534080f36b0594 columnFamilyName f 2023-07-22T12:33:30,917 DEBUG [StoreOpener-cdacde6e0baab5e76b534080f36b0594-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:33:30,917 INFO [StoreOpener-cdacde6e0baab5e76b534080f36b0594-1 {}] regionserver.HStore(324): Store=cdacde6e0baab5e76b534080f36b0594/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:33:30,918 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns3/table3_restore/cdacde6e0baab5e76b534080f36b0594 2023-07-22T12:33:30,918 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns3/table3_restore/cdacde6e0baab5e76b534080f36b0594 2023-07-22T12:33:30,921 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1076): writing seq id for cdacde6e0baab5e76b534080f36b0594 2023-07-22T12:33:30,923 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns3/table3_restore/cdacde6e0baab5e76b534080f36b0594/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:33:30,923 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1093): Opened cdacde6e0baab5e76b534080f36b0594; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10874241280, jitterRate=0.01274263858795166}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-07-22T12:33:30,924 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(992): Region open journal for cdacde6e0baab5e76b534080f36b0594: 2023-07-22T12:33:30,925 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegionServer(2552): Post open deploy tasks for ns3:table3_restore,,1690029210566.cdacde6e0baab5e76b534080f36b0594., pid=106, masterSystemTime=1690029210910 2023-07-22T12:33:30,926 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegionServer(2579): Finished post open deploy task for ns3:table3_restore,,1690029210566.cdacde6e0baab5e76b534080f36b0594. 2023-07-22T12:33:30,926 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] handler.AssignRegionHandler(158): Opened ns3:table3_restore,,1690029210566.cdacde6e0baab5e76b534080f36b0594. 2023-07-22T12:33:30,927 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=105 updating hbase:meta row=cdacde6e0baab5e76b534080f36b0594, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:33:30,930 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=106, resume processing ppid=105 2023-07-22T12:33:30,930 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=106, ppid=105, state=SUCCESS; OpenRegionProcedure cdacde6e0baab5e76b534080f36b0594, server=jenkins-hbase3.apache.org,43639,1690029056327 in 169 msec 2023-07-22T12:33:30,932 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=105, resume processing ppid=104 2023-07-22T12:33:30,932 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=105, ppid=104, state=SUCCESS; TransitRegionStateProcedure table=ns3:table3_restore, region=cdacde6e0baab5e76b534080f36b0594, ASSIGN in 328 msec 2023-07-22T12:33:30,932 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=104, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:33:30,933 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029210932"}]},"ts":"1690029210932"} 2023-07-22T12:33:30,934 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:table3_restore, state=ENABLED in hbase:meta 2023-07-22T12:33:30,936 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=104, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:33:30,937 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=104, state=SUCCESS; CreateTableProcedure table=ns3:table3_restore in 370 msec 2023-07-22T12:33:31,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=104 2023-07-22T12:33:31,182 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-07-22T12:33:31,182 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(233): Backup: backup_1690029187792 hdfs://localhost:34745/backupUT/backup_1690029187792/ns3/test-16900290656252/ 2023-07-22T12:33:31,182 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-07-22T12:33:31,185 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster$12(2663): Client=jenkins//172.31.12.81 disable ns2:table2 2023-07-22T12:33:31,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=107, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=ns2:table2 2023-07-22T12:33:31,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=107 2023-07-22T12:33:31,189 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029211189"}]},"ts":"1690029211189"} 2023-07-22T12:33:31,190 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:table2, state=DISABLING in hbase:meta 2023-07-22T12:33:31,192 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(293): Set ns2:table2 to state=DISABLING 2023-07-22T12:33:31,193 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns2:table2, region=78b093193ab7ebb2bb1fe03dad7dcd4a, UNASSIGN}] 2023-07-22T12:33:31,194 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=108, ppid=107, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns2:table2, region=78b093193ab7ebb2bb1fe03dad7dcd4a, UNASSIGN 2023-07-22T12:33:31,195 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=108 updating hbase:meta row=78b093193ab7ebb2bb1fe03dad7dcd4a, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:33:31,196 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE; CloseRegionProcedure 78b093193ab7ebb2bb1fe03dad7dcd4a, server=jenkins-hbase3.apache.org,43639,1690029056327}] 2023-07-22T12:33:31,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=107 2023-07-22T12:33:31,348 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] handler.UnassignRegionHandler(119): Close 78b093193ab7ebb2bb1fe03dad7dcd4a 2023-07-22T12:33:31,348 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegion(1668): Closing 78b093193ab7ebb2bb1fe03dad7dcd4a, disabling compactions & flushes 2023-07-22T12:33:31,348 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegion(1690): Closing region ns2:table2,,1690029206061.78b093193ab7ebb2bb1fe03dad7dcd4a. 2023-07-22T12:33:31,349 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:table2,,1690029206061.78b093193ab7ebb2bb1fe03dad7dcd4a. 2023-07-22T12:33:31,349 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegion(1778): Acquired close lock on ns2:table2,,1690029206061.78b093193ab7ebb2bb1fe03dad7dcd4a. after waiting 0 ms 2023-07-22T12:33:31,349 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegion(1788): Updates disabled for region ns2:table2,,1690029206061.78b093193ab7ebb2bb1fe03dad7dcd4a. 2023-07-22T12:33:31,350 DEBUG [StoreFileCloser-78b093193ab7ebb2bb1fe03dad7dcd4a-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 5bd07ba136024a6f9c01859bff171d6c_SeqId_4_ 2023-07-22T12:33:31,355 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/table2/78b093193ab7ebb2bb1fe03dad7dcd4a/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2023-07-22T12:33:31,355 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:33:31,356 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegion(1909): Closed ns2:table2,,1690029206061.78b093193ab7ebb2bb1fe03dad7dcd4a. 2023-07-22T12:33:31,356 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegion(1622): Region close journal for 78b093193ab7ebb2bb1fe03dad7dcd4a: 2023-07-22T12:33:31,357 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] handler.UnassignRegionHandler(163): Closed 78b093193ab7ebb2bb1fe03dad7dcd4a 2023-07-22T12:33:31,358 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=108 updating hbase:meta row=78b093193ab7ebb2bb1fe03dad7dcd4a, regionState=CLOSED 2023-07-22T12:33:31,361 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=109, resume processing ppid=108 2023-07-22T12:33:31,361 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=109, ppid=108, state=SUCCESS; CloseRegionProcedure 78b093193ab7ebb2bb1fe03dad7dcd4a, server=jenkins-hbase3.apache.org,43639,1690029056327 in 163 msec 2023-07-22T12:33:31,362 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=108, resume processing ppid=107 2023-07-22T12:33:31,362 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=108, ppid=107, state=SUCCESS; TransitRegionStateProcedure table=ns2:table2, region=78b093193ab7ebb2bb1fe03dad7dcd4a, UNASSIGN in 169 msec 2023-07-22T12:33:31,363 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029211362"}]},"ts":"1690029211362"} 2023-07-22T12:33:31,364 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:table2, state=DISABLED in hbase:meta 2023-07-22T12:33:31,368 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(305): Set ns2:table2 to state=DISABLED 2023-07-22T12:33:31,370 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=107, state=SUCCESS; DisableTableProcedure table=ns2:table2 in 184 msec 2023-07-22T12:33:31,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=107 2023-07-22T12:33:31,492 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster$5(2447): Client=jenkins//172.31.12.81 delete ns2:table2 2023-07-22T12:33:31,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=110, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=ns2:table2 2023-07-22T12:33:31,495 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=110, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=ns2:table2 2023-07-22T12:33:31,495 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=110, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=ns2:table2 2023-07-22T12:33:31,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=110 2023-07-22T12:33:31,497 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/table2/78b093193ab7ebb2bb1fe03dad7dcd4a 2023-07-22T12:33:31,499 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/table2/78b093193ab7ebb2bb1fe03dad7dcd4a/f, FileablePath, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/table2/78b093193ab7ebb2bb1fe03dad7dcd4a/recovered.edits] 2023-07-22T12:33:31,503 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/table2/78b093193ab7ebb2bb1fe03dad7dcd4a/f/5bd07ba136024a6f9c01859bff171d6c_SeqId_4_ to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/archive/data/ns2/table2/78b093193ab7ebb2bb1fe03dad7dcd4a/f/5bd07ba136024a6f9c01859bff171d6c_SeqId_4_ 2023-07-22T12:33:31,506 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/table2/78b093193ab7ebb2bb1fe03dad7dcd4a/recovered.edits/6.seqid to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/archive/data/ns2/table2/78b093193ab7ebb2bb1fe03dad7dcd4a/recovered.edits/6.seqid 2023-07-22T12:33:31,507 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/table2/78b093193ab7ebb2bb1fe03dad7dcd4a 2023-07-22T12:33:31,507 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(293): Archived ns2:table2 regions 2023-07-22T12:33:31,509 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=110, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=ns2:table2 2023-07-22T12:33:31,511 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of ns2:table2 from hbase:meta 2023-07-22T12:33:31,514 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(387): Removing 'ns2:table2' descriptor. 2023-07-22T12:33:31,515 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=110, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=ns2:table2 2023-07-22T12:33:31,515 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(377): Removing 'ns2:table2' from region states. 2023-07-22T12:33:31,515 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns2:table2,,1690029206061.78b093193ab7ebb2bb1fe03dad7dcd4a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1690029211515"}]},"ts":"9223372036854775807"} 2023-07-22T12:33:31,520 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-07-22T12:33:31,520 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 78b093193ab7ebb2bb1fe03dad7dcd4a, NAME => 'ns2:table2,,1690029206061.78b093193ab7ebb2bb1fe03dad7dcd4a.', STARTKEY => '', ENDKEY => ''}] 2023-07-22T12:33:31,520 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(381): Marking 'ns2:table2' as deleted. 2023-07-22T12:33:31,520 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1690029211520"}]},"ts":"9223372036854775807"} 2023-07-22T12:33:31,522 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1678): Deleted table ns2:table2 state from META 2023-07-22T12:33:31,525 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(127): Finished pid=110, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=ns2:table2 2023-07-22T12:33:31,526 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=110, state=SUCCESS; DeleteTableProcedure table=ns2:table2 in 33 msec 2023-07-22T12:33:31,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=110 2023-07-22T12:33:31,597 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster$12(2663): Client=jenkins//172.31.12.81 disable ns3:table3_restore 2023-07-22T12:33:31,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=111, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=ns3:table3_restore 2023-07-22T12:33:31,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=111 2023-07-22T12:33:31,601 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029211601"}]},"ts":"1690029211601"} 2023-07-22T12:33:31,602 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:table3_restore, state=DISABLING in hbase:meta 2023-07-22T12:33:31,604 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(293): Set ns3:table3_restore to state=DISABLING 2023-07-22T12:33:31,604 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns3:table3_restore, region=cdacde6e0baab5e76b534080f36b0594, UNASSIGN}] 2023-07-22T12:33:31,606 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=112, ppid=111, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns3:table3_restore, region=cdacde6e0baab5e76b534080f36b0594, UNASSIGN 2023-07-22T12:33:31,607 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=112 updating hbase:meta row=cdacde6e0baab5e76b534080f36b0594, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:33:31,608 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE; CloseRegionProcedure cdacde6e0baab5e76b534080f36b0594, server=jenkins-hbase3.apache.org,43639,1690029056327}] 2023-07-22T12:33:31,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=111 2023-07-22T12:33:31,763 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] handler.UnassignRegionHandler(119): Close cdacde6e0baab5e76b534080f36b0594 2023-07-22T12:33:31,764 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] regionserver.HRegion(1668): Closing cdacde6e0baab5e76b534080f36b0594, disabling compactions & flushes 2023-07-22T12:33:31,765 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] regionserver.HRegion(1690): Closing region ns3:table3_restore,,1690029210566.cdacde6e0baab5e76b534080f36b0594. 2023-07-22T12:33:31,765 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:table3_restore,,1690029210566.cdacde6e0baab5e76b534080f36b0594. 2023-07-22T12:33:31,765 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] regionserver.HRegion(1778): Acquired close lock on ns3:table3_restore,,1690029210566.cdacde6e0baab5e76b534080f36b0594. after waiting 0 ms 2023-07-22T12:33:31,765 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] regionserver.HRegion(1788): Updates disabled for region ns3:table3_restore,,1690029210566.cdacde6e0baab5e76b534080f36b0594. 2023-07-22T12:33:31,784 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns3/table3_restore/cdacde6e0baab5e76b534080f36b0594/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-07-22T12:33:31,785 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:33:31,785 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] regionserver.HRegion(1909): Closed ns3:table3_restore,,1690029210566.cdacde6e0baab5e76b534080f36b0594. 2023-07-22T12:33:31,785 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] regionserver.HRegion(1622): Region close journal for cdacde6e0baab5e76b534080f36b0594: 2023-07-22T12:33:31,786 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] handler.UnassignRegionHandler(163): Closed cdacde6e0baab5e76b534080f36b0594 2023-07-22T12:33:31,787 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=112 updating hbase:meta row=cdacde6e0baab5e76b534080f36b0594, regionState=CLOSED 2023-07-22T12:33:31,790 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=113, resume processing ppid=112 2023-07-22T12:33:31,790 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=113, ppid=112, state=SUCCESS; CloseRegionProcedure cdacde6e0baab5e76b534080f36b0594, server=jenkins-hbase3.apache.org,43639,1690029056327 in 180 msec 2023-07-22T12:33:31,792 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=112, resume processing ppid=111 2023-07-22T12:33:31,792 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=112, ppid=111, state=SUCCESS; TransitRegionStateProcedure table=ns3:table3_restore, region=cdacde6e0baab5e76b534080f36b0594, UNASSIGN in 187 msec 2023-07-22T12:33:31,793 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029211793"}]},"ts":"1690029211793"} 2023-07-22T12:33:31,794 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:table3_restore, state=DISABLED in hbase:meta 2023-07-22T12:33:31,796 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(305): Set ns3:table3_restore to state=DISABLED 2023-07-22T12:33:31,798 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=111, state=SUCCESS; DisableTableProcedure table=ns3:table3_restore in 198 msec 2023-07-22T12:33:31,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=111 2023-07-22T12:33:31,904 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster$5(2447): Client=jenkins//172.31.12.81 delete ns3:table3_restore 2023-07-22T12:33:31,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=114, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=ns3:table3_restore 2023-07-22T12:33:31,907 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=114, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-07-22T12:33:31,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=114 2023-07-22T12:33:31,908 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=114, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-07-22T12:33:31,910 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns3/table3_restore/cdacde6e0baab5e76b534080f36b0594 2023-07-22T12:33:31,911 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns3/table3_restore/cdacde6e0baab5e76b534080f36b0594/f, FileablePath, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns3/table3_restore/cdacde6e0baab5e76b534080f36b0594/recovered.edits] 2023-07-22T12:33:31,917 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns3/table3_restore/cdacde6e0baab5e76b534080f36b0594/recovered.edits/4.seqid to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/archive/data/ns3/table3_restore/cdacde6e0baab5e76b534080f36b0594/recovered.edits/4.seqid 2023-07-22T12:33:31,917 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns3/table3_restore/cdacde6e0baab5e76b534080f36b0594 2023-07-22T12:33:31,917 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(293): Archived ns3:table3_restore regions 2023-07-22T12:33:31,919 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=114, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-07-22T12:33:31,921 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of ns3:table3_restore from hbase:meta 2023-07-22T12:33:31,922 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(387): Removing 'ns3:table3_restore' descriptor. 2023-07-22T12:33:31,923 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=114, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-07-22T12:33:31,923 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(377): Removing 'ns3:table3_restore' from region states. 2023-07-22T12:33:31,923 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns3:table3_restore,,1690029210566.cdacde6e0baab5e76b534080f36b0594.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1690029211923"}]},"ts":"9223372036854775807"} 2023-07-22T12:33:31,925 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-07-22T12:33:31,925 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => cdacde6e0baab5e76b534080f36b0594, NAME => 'ns3:table3_restore,,1690029210566.cdacde6e0baab5e76b534080f36b0594.', STARTKEY => '', ENDKEY => ''}] 2023-07-22T12:33:31,925 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(381): Marking 'ns3:table3_restore' as deleted. 2023-07-22T12:33:31,925 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1690029211925"}]},"ts":"9223372036854775807"} 2023-07-22T12:33:31,926 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1678): Deleted table ns3:table3_restore state from META 2023-07-22T12:33:31,928 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(127): Finished pid=114, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-07-22T12:33:31,928 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=114, state=SUCCESS; DeleteTableProcedure table=ns3:table3_restore in 24 msec 2023-07-22T12:33:32,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=114 2023-07-22T12:33:32,025 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreMultiple Thread=601 (was 597) - Thread LEAK? -, OpenFileDescriptor=681 (was 682), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=154 (was 112) - SystemLoadAverage LEAK? -, ProcessCount=169 (was 169), AvailableMemoryMB=16984 (was 16107) - AvailableMemoryMB LEAK? - 2023-07-22T12:33:32,025 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(130): Thread=601 is superior to 500 2023-07-22T12:33:32,043 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreMultipleOverwriteCommand Thread=601, OpenFileDescriptor=681, MaxFileDescriptor=60000, SystemLoadAverage=154, ProcessCount=169, AvailableMemoryMB=16983 2023-07-22T12:33:32,043 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(130): Thread=601 is superior to 500 2023-07-22T12:33:32,043 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] backup.TestFullRestore(234): create full backup image on multiple tables: command-line 2023-07-22T12:33:32,049 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@240a06df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:33:32,053 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:33:32,055 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:37464, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:33:32,057 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:33:32,059 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:36626, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:33:32,069 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-07-22T12:33:32,073 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:33:32,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029212073 (current time:1690029212073). 2023-07-22T12:33:32,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:33:32,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-07-22T12:33:32,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1839df9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:33:32,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:33:32,081 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:37468, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:33:32,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:33:32,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:33:32,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=115, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:33:32,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 115 2023-07-22T12:33:32,086 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:33:32,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=115 2023-07-22T12:33:32,087 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:33:32,088 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:33:32,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=115 2023-07-22T12:33:32,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=115 2023-07-22T12:33:32,497 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:33:32,500 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33}] 2023-07-22T12:33:32,502 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=116, ppid=115, state=RUNNABLE; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33 2023-07-22T12:33:32,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43639 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=116 2023-07-22T12:33:32,654 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. 2023-07-22T12:33:32,654 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.HRegion(2819): Flushing 697cc5ecf0962f5ab9f4126febf7ce33 2/2 column families, dataSize=1.50 KB heapSize=2.73 KB 2023-07-22T12:33:32,658 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/meta/3e46ffe2abec4ceba3ab7424eff1a910 is 171, key is trslm:hdfs://localhost:34745/backupUT\x00ns2:test-16900290656251/meta:log-roll-map/1690029205594/Put/seqid=0 2023-07-22T12:33:32,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=115 2023-07-22T12:33:32,742 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-07-22T12:33:33,066 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=717 B at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/meta/3e46ffe2abec4ceba3ab7424eff1a910 2023-07-22T12:33:33,071 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 3e46ffe2abec4ceba3ab7424eff1a910 2023-07-22T12:33:33,074 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/session/83933de825724e92a18f2bfe93b821b6 is 524, key is session:backup_1690029187792/session:context/1690029206025/Put/seqid=0 2023-07-22T12:33:33,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=115 2023-07-22T12:33:33,478 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=820 B at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/session/83933de825724e92a18f2bfe93b821b6 2023-07-22T12:33:33,483 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 83933de825724e92a18f2bfe93b821b6 2023-07-22T12:33:33,483 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/meta/3e46ffe2abec4ceba3ab7424eff1a910 as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/3e46ffe2abec4ceba3ab7424eff1a910 2023-07-22T12:33:33,487 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/3e46ffe2abec4ceba3ab7424eff1a910, entries=6, sequenceid=94, filesize=5.8 K 2023-07-22T12:33:33,488 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/session/83933de825724e92a18f2bfe93b821b6 as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/83933de825724e92a18f2bfe93b821b6 2023-07-22T12:33:33,492 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/83933de825724e92a18f2bfe93b821b6, entries=2, sequenceid=94, filesize=5.5 K 2023-07-22T12:33:33,493 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.HRegion(3022): Finished flush of dataSize ~1.50 KB/1537, heapSize ~2.70 KB/2760, currentSize=0 B/0 for 697cc5ecf0962f5ab9f4126febf7ce33 in 839ms, sequenceid=94, compaction requested=true 2023-07-22T12:33:33,493 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.HRegion(2520): Flush status journal for 697cc5ecf0962f5ab9f4126febf7ce33: 2023-07-22T12:33:33,493 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. for snapshot_backup_system completed. 2023-07-22T12:33:33,493 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33.' region-info for snapshot=snapshot_backup_system 2023-07-22T12:33:33,493 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:33:33,493 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/b580ac8bece04b07a21605e555bdc696, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/5ea0158e2d574f758c68ee9c20c21191, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/a6d2695c5574440ea495f58a7c66ed03, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/8f78c6cced064299a30d72a0b4901528, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/606f117b1e384a54bb6dba5339e90e8f, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/15bf4f9b1b65467c96713e751468dc6c, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/b30f291cd15a4d92ac4b31f477e93428, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/3e46ffe2abec4ceba3ab7424eff1a910] hfiles 2023-07-22T12:33:33,493 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (1/8): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/b580ac8bece04b07a21605e555bdc696 for snapshot=snapshot_backup_system 2023-07-22T12:33:33,493 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (2/8): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/5ea0158e2d574f758c68ee9c20c21191 for snapshot=snapshot_backup_system 2023-07-22T12:33:33,493 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (3/8): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/a6d2695c5574440ea495f58a7c66ed03 for snapshot=snapshot_backup_system 2023-07-22T12:33:33,493 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (4/8): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/8f78c6cced064299a30d72a0b4901528 for snapshot=snapshot_backup_system 2023-07-22T12:33:33,493 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (5/8): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/606f117b1e384a54bb6dba5339e90e8f for snapshot=snapshot_backup_system 2023-07-22T12:33:33,493 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (6/8): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/15bf4f9b1b65467c96713e751468dc6c for snapshot=snapshot_backup_system 2023-07-22T12:33:33,493 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (7/8): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/b30f291cd15a4d92ac4b31f477e93428 for snapshot=snapshot_backup_system 2023-07-22T12:33:33,494 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (8/8): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/3e46ffe2abec4ceba3ab7424eff1a910 for snapshot=snapshot_backup_system 2023-07-22T12:33:33,494 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/956010b3eaa64b4fa82ddea4557fac2c, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/0704967b71ce4a5cb0af553e6e47addb, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/62b5d2f1465b492ab802e26f5bf32f6b, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/c2fdeacbf4184719a11904adfeaf1fd8, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/660347589bb3465a82d9c9138151d6a4, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/2b4c40a0167b43f584d34520d18e9390, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/1550986ca81f443894a286cada73692c, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/a5d57e73d32d4eda8e52490df82775fa, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/83933de825724e92a18f2bfe93b821b6] hfiles 2023-07-22T12:33:33,494 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (1/9): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/956010b3eaa64b4fa82ddea4557fac2c for snapshot=snapshot_backup_system 2023-07-22T12:33:33,494 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (2/9): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/0704967b71ce4a5cb0af553e6e47addb for snapshot=snapshot_backup_system 2023-07-22T12:33:33,494 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (3/9): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/62b5d2f1465b492ab802e26f5bf32f6b for snapshot=snapshot_backup_system 2023-07-22T12:33:33,494 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (4/9): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/c2fdeacbf4184719a11904adfeaf1fd8 for snapshot=snapshot_backup_system 2023-07-22T12:33:33,494 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (5/9): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/660347589bb3465a82d9c9138151d6a4 for snapshot=snapshot_backup_system 2023-07-22T12:33:33,494 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (6/9): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/2b4c40a0167b43f584d34520d18e9390 for snapshot=snapshot_backup_system 2023-07-22T12:33:33,494 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (7/9): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/1550986ca81f443894a286cada73692c for snapshot=snapshot_backup_system 2023-07-22T12:33:33,494 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (8/9): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/a5d57e73d32d4eda8e52490df82775fa for snapshot=snapshot_backup_system 2023-07-22T12:33:33,494 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (9/9): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/83933de825724e92a18f2bfe93b821b6 for snapshot=snapshot_backup_system 2023-07-22T12:33:33,511 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. 2023-07-22T12:33:33,511 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=116 2023-07-22T12:33:33,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster(3994): Remote procedure done, pid=116 2023-07-22T12:33:33,512 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 697cc5ecf0962f5ab9f4126febf7ce33 2023-07-22T12:33:33,513 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=116, ppid=115, state=RUNNABLE; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33 2023-07-22T12:33:33,516 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=116, resume processing ppid=115 2023-07-22T12:33:33,516 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:33:33,516 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=116, ppid=115, state=SUCCESS; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33 in 1.0160 sec 2023-07-22T12:33:33,516 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:33:33,517 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:33:33,517 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-07-22T12:33:33,517 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_backup_system 2023-07-22T12:33:33,929 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:33:33,941 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:33:33,941 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_backup_system 2023-07-22T12:33:34,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=115 2023-07-22T12:33:34,760 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:33:34,760 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 115 2023-07-22T12:33:34,761 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=115, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.6780 sec 2023-07-22T12:33:36,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=115 2023-07-22T12:33:36,193 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(115): Backup backup_1690029212049 started at 1690029216193. 2023-07-22T12:33:36,194 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(119): Backup session backup_1690029212049 has been started. 2023-07-22T12:33:36,199 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-07-22T12:33:36,200 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-07-22T12:33:36,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-07-22T12:33:36,200 INFO [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-07-22T12:33:36,200 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-07-22T12:33:36,201 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-07-22T12:33:36,201 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-07-22T12:33:36,202 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:33:36,204 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:33:36,204 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:33:36,205 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:33:36,205 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-07-22T12:33:36,205 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-07-22T12:33:36,206 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-07-22T12:33:36,206 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:33:36,206 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-07-22T12:33:36,206 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-07-22T12:33:36,206 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-07-22T12:33:36,206 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-07-22T12:33:36,206 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-07-22T12:33:36,207 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-07-22T12:33:36,207 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-07-22T12:33:36,207 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,43639,1690029056327' joining acquired barrier for procedure (rolllog) in zk 2023-07-22T12:33:36,209 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-07-22T12:33:36,210 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:33:36,210 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:33:36,210 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:33:36,210 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-07-22T12:33:36,210 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:33:36,210 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:33:36,210 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:33:36,211 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:33:36,211 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:33:36,211 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,43639,1690029056327' joining acquired barrier for procedure 'rolllog' on coordinator 2023-07-22T12:33:36,211 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@7fed8c3f[Count = 0] remaining members to acquire global barrier 2023-07-22T12:33:36,211 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-07-22T12:33:36,211 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-07-22T12:33:36,213 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-07-22T12:33:36,213 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-07-22T12:33:36,213 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-07-22T12:33:36,213 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-07-22T12:33:36,214 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-07-22T12:33:36,214 DEBUG [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:33:36,214 INFO [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1690029191975 highest: 1690029191975 on jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:33:36,214 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta:.meta(num 1690029191958) roll requested 2023-07-22T12:33:36,225 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029216214.meta, exclude list is [], retry=0 2023-07-22T12:33:36,229 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK] 2023-07-22T12:33:36,231 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029191958.meta with entries=24, filesize=5.75 KB; new WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029216214.meta 2023-07-22T12:33:36,231 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK]] 2023-07-22T12:33:36,231 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029191958.meta is not closed yet, will try archiving it next time 2023-07-22T12:33:36,231 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C43639%2C1690029056327:(num 1690029191975) roll requested 2023-07-22T12:33:36,234 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029191958.meta not finished, retry = 0 2023-07-22T12:33:36,243 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029216231, exclude list is [], retry=0 2023-07-22T12:33:36,247 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK] 2023-07-22T12:33:36,249 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029191975 with entries=15, filesize=4.08 KB; new WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029216231 2023-07-22T12:33:36,249 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK]] 2023-07-22T12:33:36,249 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029191975 is not closed yet, will try archiving it next time 2023-07-22T12:33:36,249 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029175708 to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/oldWALs/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029175708 2023-07-22T12:33:36,252 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029191975 not finished, retry = 0 2023-07-22T12:33:36,254 DEBUG [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-07-22T12:33:36,254 INFO [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1690029216231 on jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:33:36,264 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-07-22T12:33:36,264 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,43639,1690029056327' in zk 2023-07-22T12:33:36,267 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-07-22T12:33:36,267 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:33:36,267 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:33:36,268 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-07-22T12:33:36,269 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:33:36,269 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:33:36,269 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:33:36,269 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:33:36,270 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:33:36,270 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:33:36,270 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:33:36,271 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:33:36,271 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:33:36,272 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,43639,1690029056327': 2023-07-22T12:33:36,272 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,43639,1690029056327' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-07-22T12:33:36,272 INFO [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-07-22T12:33:36,272 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-07-22T12:33:36,272 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-07-22T12:33:36,272 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-07-22T12:33:36,272 INFO [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-07-22T12:33:36,275 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-07-22T12:33:36,275 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-07-22T12:33:36,275 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:33:36,275 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:33:36,275 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:33:36,275 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:33:36,275 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:33:36,275 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:33:36,276 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:33:36,276 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:33:36,276 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:33:36,276 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:33:36,277 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:33:36,277 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:33:36,277 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:33:36,278 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:33:36,284 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:33:36,284 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:33:36,284 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:33:36,284 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-07-22T12:33:36,285 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-07-22T12:33:36,285 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:33:36,285 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:33:36,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-07-22T12:33:36,389 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1690029216389_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } 2023-07-22T12:33:36,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029216389 (current time:1690029216389). 2023-07-22T12:33:36,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:33:36,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1690029216389_ns2_test-16900290656251 VERSION not specified, setting to 2 2023-07-22T12:33:36,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61e80bed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:33:36,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:33:36,398 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:37472, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:33:36,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:33:36,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:33:36,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=117, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1690029216389_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } 2023-07-22T12:33:36,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1690029216389_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 }, snapshot procedure id = 117 2023-07-22T12:33:36,401 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1690029216389_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:33:36,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=117 2023-07-22T12:33:36,402 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1690029216389_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:33:36,403 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1690029216389_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:33:36,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=117 2023-07-22T12:33:36,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=117 2023-07-22T12:33:36,809 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1690029216389_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:33:36,809 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE; SnapshotRegionProcedure b8196e3466217e5359c3acacaaeb9e61}] 2023-07-22T12:33:36,811 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=118, ppid=117, state=RUNNABLE; SnapshotRegionProcedure b8196e3466217e5359c3acacaaeb9e61 2023-07-22T12:33:36,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43639 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=118 2023-07-22T12:33:36,963 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-16900290656251,,1690029096058.b8196e3466217e5359c3acacaaeb9e61. 2023-07-22T12:33:36,963 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] regionserver.HRegion(2520): Flush status journal for b8196e3466217e5359c3acacaaeb9e61: 2023-07-22T12:33:36,963 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-16900290656251,,1690029096058.b8196e3466217e5359c3acacaaeb9e61. for snapshot_1690029216389_ns2_test-16900290656251 completed. 2023-07-22T12:33:36,963 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] snapshot.SnapshotManifest(240): Storing 'ns2:test-16900290656251,,1690029096058.b8196e3466217e5359c3acacaaeb9e61.' region-info for snapshot=snapshot_1690029216389_ns2_test-16900290656251 2023-07-22T12:33:36,963 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:33:36,963 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/b8196e3466217e5359c3acacaaeb9e61/f/82851155b50f47edb054b7e8d9f06090_SeqId_4_] hfiles 2023-07-22T12:33:36,963 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/b8196e3466217e5359c3acacaaeb9e61/f/82851155b50f47edb054b7e8d9f06090_SeqId_4_ for snapshot=snapshot_1690029216389_ns2_test-16900290656251 2023-07-22T12:33:37,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=117 2023-07-22T12:33:37,370 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-16900290656251,,1690029096058.b8196e3466217e5359c3acacaaeb9e61. 2023-07-22T12:33:37,370 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=118 2023-07-22T12:33:37,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster(3994): Remote procedure done, pid=118 2023-07-22T12:33:37,370 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1690029216389_ns2_test-16900290656251 on region b8196e3466217e5359c3acacaaeb9e61 2023-07-22T12:33:37,372 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=118, ppid=117, state=RUNNABLE; SnapshotRegionProcedure b8196e3466217e5359c3acacaaeb9e61 2023-07-22T12:33:37,374 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=118, resume processing ppid=117 2023-07-22T12:33:37,374 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1690029216389_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:33:37,374 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=118, ppid=117, state=SUCCESS; SnapshotRegionProcedure b8196e3466217e5359c3acacaaeb9e61 in 563 msec 2023-07-22T12:33:37,375 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1690029216389_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:33:37,375 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1690029216389_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:33:37,375 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1690029216389_ns2_test-16900290656251 2023-07-22T12:33:37,376 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_1690029216389_ns2_test-16900290656251 2023-07-22T12:33:37,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=117 2023-07-22T12:33:37,785 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1690029216389_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:33:37,790 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1690029216389_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:33:37,790 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_1690029216389_ns2_test-16900290656251 to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029216389_ns2_test-16900290656251 2023-07-22T12:33:38,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=117 2023-07-22T12:33:38,606 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1690029216389_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:33:38,606 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1690029216389_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 }, snapshot procedure id = 117 2023-07-22T12:33:38,610 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=117, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1690029216389_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } in 2.2070 sec 2023-07-22T12:33:40,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=117 2023-07-22T12:33:40,509 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1690029220509_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } 2023-07-22T12:33:40,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029220509 (current time:1690029220509). 2023-07-22T12:33:40,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:33:40,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1690029220509_ns3_test-16900290656252 VERSION not specified, setting to 2 2023-07-22T12:33:40,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@283b8915, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:33:40,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:33:40,517 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:55520, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:33:40,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:33:40,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:33:40,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=119, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1690029220509_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } 2023-07-22T12:33:40,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1690029220509_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 }, snapshot procedure id = 119 2023-07-22T12:33:40,520 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1690029220509_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:33:40,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=119 2023-07-22T12:33:40,521 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1690029220509_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:33:40,522 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1690029220509_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:33:40,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=119 2023-07-22T12:33:40,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=119 2023-07-22T12:33:40,927 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1690029220509_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:33:40,928 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE; SnapshotRegionProcedure aefea80783ea312734eb5d79ad00b2f1}] 2023-07-22T12:33:40,931 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=120, ppid=119, state=RUNNABLE; SnapshotRegionProcedure aefea80783ea312734eb5d79ad00b2f1 2023-07-22T12:33:41,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43639 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=120 2023-07-22T12:33:41,082 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=120}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-16900290656252,,1690029100393.aefea80783ea312734eb5d79ad00b2f1. 2023-07-22T12:33:41,083 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=120}] regionserver.HRegion(2520): Flush status journal for aefea80783ea312734eb5d79ad00b2f1: 2023-07-22T12:33:41,083 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=120}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-16900290656252,,1690029100393.aefea80783ea312734eb5d79ad00b2f1. for snapshot_1690029220509_ns3_test-16900290656252 completed. 2023-07-22T12:33:41,083 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=120}] snapshot.SnapshotManifest(240): Storing 'ns3:test-16900290656252,,1690029100393.aefea80783ea312734eb5d79ad00b2f1.' region-info for snapshot=snapshot_1690029220509_ns3_test-16900290656252 2023-07-22T12:33:41,083 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=120}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:33:41,083 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=120}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-07-22T12:33:41,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=119 2023-07-22T12:33:41,488 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=120}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-16900290656252,,1690029100393.aefea80783ea312734eb5d79ad00b2f1. 2023-07-22T12:33:41,488 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=120 2023-07-22T12:33:41,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster(3994): Remote procedure done, pid=120 2023-07-22T12:33:41,489 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1690029220509_ns3_test-16900290656252 on region aefea80783ea312734eb5d79ad00b2f1 2023-07-22T12:33:41,491 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=120, ppid=119, state=RUNNABLE; SnapshotRegionProcedure aefea80783ea312734eb5d79ad00b2f1 2023-07-22T12:33:41,492 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=120, resume processing ppid=119 2023-07-22T12:33:41,492 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=120, ppid=119, state=SUCCESS; SnapshotRegionProcedure aefea80783ea312734eb5d79ad00b2f1 in 563 msec 2023-07-22T12:33:41,492 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1690029220509_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:33:41,493 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1690029220509_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:33:41,493 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1690029220509_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:33:41,493 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1690029220509_ns3_test-16900290656252 2023-07-22T12:33:41,494 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_1690029220509_ns3_test-16900290656252 2023-07-22T12:33:41,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=119 2023-07-22T12:33:41,903 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1690029220509_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:33:41,908 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1690029220509_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:33:41,909 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_1690029220509_ns3_test-16900290656252 to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029220509_ns3_test-16900290656252 2023-07-22T12:33:42,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=119 2023-07-22T12:33:42,734 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1690029220509_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:33:42,734 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1690029220509_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 }, snapshot procedure id = 119 2023-07-22T12:33:42,735 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=119, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1690029220509_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } in 2.2160 sec 2023-07-22T12:33:44,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=119 2023-07-22T12:33:44,627 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1690029212049 2023-07-22T12:33:44,627 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-07-22T12:33:44,627 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2023-07-22T12:33:44,627 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1690029212049_ns2:test-16900290656251 2023-07-22T12:33:44,627 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1690029216389_ns2_test-16900290656251 to hdfs://localhost:34745/backupUT/backup_1690029212049/ns2/test-16900290656251/ 2023-07-22T12:33:44,627 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-07-22T12:33:44,652 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:34745, inputRoot=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947 2023-07-22T12:33:44,652 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,37923,1690029053050_982520955_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:34745/backupUT/backup_1690029212049/ns2/test-16900290656251, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:34745/backupUT/backup_1690029212049/ns2/test-16900290656251/.hbase-snapshot/.tmp/snapshot_1690029216389_ns2_test-16900290656251 2023-07-22T12:33:44,652 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:34745, inputRoot=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947, snapshotDir=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029216389_ns2_test-16900290656251. 2023-07-22T12:33:44,660 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029216389_ns2_test-16900290656251 to hdfs://localhost:34745/backupUT/backup_1690029212049/ns2/test-16900290656251/.hbase-snapshot/.tmp/snapshot_1690029216389_ns2_test-16900290656251 2023-07-22T12:33:45,080 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:45,080 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:45,081 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:45,081 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:46,115 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/hadoop-1599149223836844262.jar 2023-07-22T12:33:46,115 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:46,115 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:46,116 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:46,116 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:46,116 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:46,117 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:46,117 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:46,117 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:46,118 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:46,118 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:33:46,119 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-07-22T12:33:46,119 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-07-22T12:33:46,120 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-07-22T12:33:46,120 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-07-22T12:33:46,120 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-07-22T12:33:46,121 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-07-22T12:33:46,121 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-07-22T12:33:46,122 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-07-22T12:33:46,122 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-07-22T12:33:46,122 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-07-22T12:33:46,123 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-07-22T12:33:46,123 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:33:46,124 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:33:46,124 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:33:46,124 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:33:46,125 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:33:46,125 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:33:46,126 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:33:46,172 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1690029216389_ns2_test-16900290656251' hfile list 2023-07-22T12:33:46,174 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-07-22T12:33:46,244 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029226210/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-07-22T12:33:46,244 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029226210/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-07-22T12:33:46,298 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-07-22T12:33:46,343 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/b8196e3466217e5359c3acacaaeb9e61/f/82851155b50f47edb054b7e8d9f06090_SeqId_4_ output=hdfs://localhost:34745/backupUT/backup_1690029212049/ns2/test-16900290656251/archive/data/ns2/test-16900290656251/b8196e3466217e5359c3acacaaeb9e61/f/82851155b50f47edb054b7e8d9f06090_SeqId_4_ 2023-07-22T12:33:46,343 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8725 (8.5 K) time=0sec 8.321M/sec 2023-07-22T12:33:47,281 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-07-22T12:33:47,282 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-07-22T12:33:47,287 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1690029216389_ns2_test-16900290656251 2023-07-22T12:33:47,287 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1690029216389_ns2_test-16900290656251 finished. 2023-07-22T12:33:47,287 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1690029212049_ns3:test-16900290656252 2023-07-22T12:33:47,287 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1690029220509_ns3_test-16900290656252 to hdfs://localhost:34745/backupUT/backup_1690029212049/ns3/test-16900290656252/ 2023-07-22T12:33:47,287 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-07-22T12:33:47,310 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:34745, inputRoot=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947 2023-07-22T12:33:47,310 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,37923,1690029053050_1115864441_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:34745/backupUT/backup_1690029212049/ns3/test-16900290656252, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:34745/backupUT/backup_1690029212049/ns3/test-16900290656252/.hbase-snapshot/.tmp/snapshot_1690029220509_ns3_test-16900290656252 2023-07-22T12:33:47,310 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:34745, inputRoot=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947, snapshotDir=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029220509_ns3_test-16900290656252. 2023-07-22T12:33:47,318 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029220509_ns3_test-16900290656252 to hdfs://localhost:34745/backupUT/backup_1690029212049/ns3/test-16900290656252/.hbase-snapshot/.tmp/snapshot_1690029220509_ns3_test-16900290656252 2023-07-22T12:33:47,734 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:47,734 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:47,735 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:47,735 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:48,719 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/hadoop-1029739800272611623.jar 2023-07-22T12:33:48,719 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:48,720 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:48,720 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:48,720 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:48,721 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:48,721 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:48,721 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:48,722 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:48,722 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:48,723 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:33:48,723 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-07-22T12:33:48,723 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-07-22T12:33:48,724 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-07-22T12:33:48,724 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-07-22T12:33:48,725 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-07-22T12:33:48,725 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-07-22T12:33:48,725 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-07-22T12:33:48,726 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-07-22T12:33:48,726 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-07-22T12:33:48,726 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-07-22T12:33:48,727 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-07-22T12:33:48,727 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:33:48,728 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:33:48,728 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:33:48,728 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:33:48,729 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:33:48,729 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:33:48,730 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:33:48,772 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1690029220509_ns3_test-16900290656252' hfile list 2023-07-22T12:33:48,839 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029228808/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-07-22T12:33:48,839 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029228808/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-07-22T12:33:49,866 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-07-22T12:33:49,867 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-07-22T12:33:49,872 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1690029220509_ns3_test-16900290656252 2023-07-22T12:33:49,872 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1690029220509_ns3_test-16900290656252 finished. 2023-07-22T12:33:49,872 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(1014): ns2:test-16900290656251 2023-07-22T12:33:49,872 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(1014): ns3:test-16900290656252 2023-07-22T12:33:49,878 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029212049 2023-07-22T12:33:49,878 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-07-22T12:33:50,282 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:34745/backupUT/backup_1690029212049/.backup.manifest 2023-07-22T12:33:50,282 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029212049 2023-07-22T12:33:50,282 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-07-22T12:33:50,687 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:34745/backupUT/backup_1690029212049/.backup.manifest 2023-07-22T12:33:50,688 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(358): Backup backup_1690029212049 finished: type=FULL,tablelist=ns2:test-16900290656251;ns3:test-16900290656252,targetRootDir=hdfs://localhost:34745/backupUT,startts=1690029216193,completets=1690029229878,bytescopied=0 2023-07-22T12:33:50,688 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-07-22T12:33:50,688 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1690029216389_ns2_test-16900290656251 2023-07-22T12:33:50,688 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1690029216389_ns2_test-16900290656251" 2023-07-22T12:33:50,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1690029216389_ns2_test-16900290656251 2023-07-22T12:33:50,691 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1690029216389_ns2_test-16900290656251 for backup backup_1690029212049 succeeded. 2023-07-22T12:33:50,691 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1690029220509_ns3_test-16900290656252 2023-07-22T12:33:50,692 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1690029220509_ns3_test-16900290656252" 2023-07-22T12:33:50,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1690029220509_ns3_test-16900290656252 2023-07-22T12:33:50,694 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1690029220509_ns3_test-16900290656252 for backup backup_1690029212049 succeeded. 2023-07-22T12:33:50,695 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-07-22T12:33:50,698 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-07-22T12:33:50,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-07-22T12:33:50,700 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-07-22T12:33:50,701 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-07-22T12:33:50,706 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(376): Backup backup_1690029212049 completed. 2023-07-22T12:33:50,706 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:33:50,715 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] backup.RestoreDriver(101): Found -overwrite option in restore command, will overwrite to existing table if any in the restore target 2023-07-22T12:33:50,720 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16ee5e88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:33:50,722 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:33:50,723 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:55592, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:33:50,727 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34745/backupUT/backup_1690029212049/.backup.manifest 2023-07-22T12:33:50,728 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029212049/.backup.manifest 2023-07-22T12:33:50,729 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34745/backupUT/backup_1690029212049/.backup.manifest 2023-07-22T12:33:50,730 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029212049/.backup.manifest 2023-07-22T12:33:50,731 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34745/backupUT/backup_1690029212049/.backup.manifest 2023-07-22T12:33:50,732 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029212049/.backup.manifest 2023-07-22T12:33:50,732 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(148): Restoring 'ns2:test-16900290656251' to 'ns2:test-16900290656251' from full backup image hdfs://localhost:34745/backupUT/backup_1690029212049/ns2/test-16900290656251 2023-07-22T12:33:50,738 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] util.RestoreTool(476): Truncating exising target table 'ns2:test-16900290656251', preserving region splits 2023-07-22T12:33:50,738 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:33:50,741 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:37548, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:33:50,742 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster$12(2663): Client=jenkins//172.31.12.81 disable ns2:test-16900290656251 2023-07-22T12:33:50,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=121, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=ns2:test-16900290656251 2023-07-22T12:33:50,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=121 2023-07-22T12:33:50,745 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16900290656251","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029230745"}]},"ts":"1690029230745"} 2023-07-22T12:33:50,746 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16900290656251, state=DISABLING in hbase:meta 2023-07-22T12:33:50,747 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(293): Set ns2:test-16900290656251 to state=DISABLING 2023-07-22T12:33:50,748 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns2:test-16900290656251, region=b8196e3466217e5359c3acacaaeb9e61, UNASSIGN}] 2023-07-22T12:33:50,749 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=122, ppid=121, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns2:test-16900290656251, region=b8196e3466217e5359c3acacaaeb9e61, UNASSIGN 2023-07-22T12:33:50,749 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=122 updating hbase:meta row=b8196e3466217e5359c3acacaaeb9e61, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:33:50,750 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=123, ppid=122, state=RUNNABLE; CloseRegionProcedure b8196e3466217e5359c3acacaaeb9e61, server=jenkins-hbase3.apache.org,43639,1690029056327}] 2023-07-22T12:33:50,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=121 2023-07-22T12:33:50,902 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] handler.UnassignRegionHandler(119): Close b8196e3466217e5359c3acacaaeb9e61 2023-07-22T12:33:50,902 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(1668): Closing b8196e3466217e5359c3acacaaeb9e61, disabling compactions & flushes 2023-07-22T12:33:50,902 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(1690): Closing region ns2:test-16900290656251,,1690029096058.b8196e3466217e5359c3acacaaeb9e61. 2023-07-22T12:33:50,902 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16900290656251,,1690029096058.b8196e3466217e5359c3acacaaeb9e61. 2023-07-22T12:33:50,902 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16900290656251,,1690029096058.b8196e3466217e5359c3acacaaeb9e61. after waiting 0 ms 2023-07-22T12:33:50,902 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16900290656251,,1690029096058.b8196e3466217e5359c3acacaaeb9e61. 2023-07-22T12:33:50,903 DEBUG [StoreFileCloser-b8196e3466217e5359c3acacaaeb9e61-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 82851155b50f47edb054b7e8d9f06090_SeqId_4_ 2023-07-22T12:33:50,907 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/b8196e3466217e5359c3acacaaeb9e61/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2023-07-22T12:33:50,908 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:33:50,908 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(1909): Closed ns2:test-16900290656251,,1690029096058.b8196e3466217e5359c3acacaaeb9e61. 2023-07-22T12:33:50,908 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(1622): Region close journal for b8196e3466217e5359c3acacaaeb9e61: 2023-07-22T12:33:50,910 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] handler.UnassignRegionHandler(163): Closed b8196e3466217e5359c3acacaaeb9e61 2023-07-22T12:33:50,910 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=122 updating hbase:meta row=b8196e3466217e5359c3acacaaeb9e61, regionState=CLOSED 2023-07-22T12:33:50,912 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=123, resume processing ppid=122 2023-07-22T12:33:50,912 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=123, ppid=122, state=SUCCESS; CloseRegionProcedure b8196e3466217e5359c3acacaaeb9e61, server=jenkins-hbase3.apache.org,43639,1690029056327 in 161 msec 2023-07-22T12:33:50,913 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=122, resume processing ppid=121 2023-07-22T12:33:50,913 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=122, ppid=121, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-16900290656251, region=b8196e3466217e5359c3acacaaeb9e61, UNASSIGN in 164 msec 2023-07-22T12:33:50,913 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16900290656251","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029230913"}]},"ts":"1690029230913"} 2023-07-22T12:33:50,914 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16900290656251, state=DISABLED in hbase:meta 2023-07-22T12:33:50,917 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(305): Set ns2:test-16900290656251 to state=DISABLED 2023-07-22T12:33:50,918 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=121, state=SUCCESS; DisableTableProcedure table=ns2:test-16900290656251 in 175 msec 2023-07-22T12:33:51,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=121 2023-07-22T12:33:51,047 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster$6(2479): Client=jenkins//172.31.12.81 truncate ns2:test-16900290656251 2023-07-22T12:33:51,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=124, state=RUNNABLE:TRUNCATE_TABLE_PRE_OPERATION; TruncateTableProcedure (table=ns2:test-16900290656251 preserveSplits=true) 2023-07-22T12:33:51,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=124 2023-07-22T12:33:51,050 DEBUG [PEWorker-2 {}] procedure.TruncateTableProcedure(87): waiting for 'ns2:test-16900290656251' regions in transition 2023-07-22T12:33:51,052 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/b8196e3466217e5359c3acacaaeb9e61 2023-07-22T12:33:51,053 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/b8196e3466217e5359c3acacaaeb9e61/f, FileablePath, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/b8196e3466217e5359c3acacaaeb9e61/recovered.edits] 2023-07-22T12:33:51,057 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/b8196e3466217e5359c3acacaaeb9e61/f/82851155b50f47edb054b7e8d9f06090_SeqId_4_ to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/archive/data/ns2/test-16900290656251/b8196e3466217e5359c3acacaaeb9e61/f/82851155b50f47edb054b7e8d9f06090_SeqId_4_ 2023-07-22T12:33:51,059 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/b8196e3466217e5359c3acacaaeb9e61/recovered.edits/9.seqid to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/archive/data/ns2/test-16900290656251/b8196e3466217e5359c3acacaaeb9e61/recovered.edits/9.seqid 2023-07-22T12:33:51,060 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/b8196e3466217e5359c3acacaaeb9e61 2023-07-22T12:33:51,060 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(293): Archived ns2:test-16900290656251 regions 2023-07-22T12:33:51,063 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of ns2:test-16900290656251 from hbase:meta 2023-07-22T12:33:51,065 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(387): Removing 'ns2:test-16900290656251' descriptor. 2023-07-22T12:33:51,065 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(377): Removing 'ns2:test-16900290656251' from region states. 2023-07-22T12:33:51,065 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns2:test-16900290656251,,1690029096058.b8196e3466217e5359c3acacaaeb9e61.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1690029231065"}]},"ts":"9223372036854775807"} 2023-07-22T12:33:51,067 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-07-22T12:33:51,067 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => b8196e3466217e5359c3acacaaeb9e61, NAME => 'ns2:test-16900290656251,,1690029096058.b8196e3466217e5359c3acacaaeb9e61.', STARTKEY => '', ENDKEY => ''}] 2023-07-22T12:33:51,067 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(381): Marking 'ns2:test-16900290656251' as deleted. 2023-07-22T12:33:51,067 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns2:test-16900290656251","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1690029231067"}]},"ts":"9223372036854775807"} 2023-07-22T12:33:51,068 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1678): Deleted table ns2:test-16900290656251 state from META 2023-07-22T12:33:51,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=124 2023-07-22T12:33:51,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=124 2023-07-22T12:33:51,476 INFO [RegionOpenAndInit-ns2:test-16900290656251-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 01e992ceb654b1e583b990da186e85a8, NAME => 'ns2:test-16900290656251,,1690029231061.01e992ceb654b1e583b990da186e85a8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-16900290656251', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947 2023-07-22T12:33:51,482 DEBUG [RegionOpenAndInit-ns2:test-16900290656251-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:test-16900290656251,,1690029231061.01e992ceb654b1e583b990da186e85a8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:33:51,482 DEBUG [RegionOpenAndInit-ns2:test-16900290656251-pool-0 {}] regionserver.HRegion(1668): Closing 01e992ceb654b1e583b990da186e85a8, disabling compactions & flushes 2023-07-22T12:33:51,482 INFO [RegionOpenAndInit-ns2:test-16900290656251-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:test-16900290656251,,1690029231061.01e992ceb654b1e583b990da186e85a8. 2023-07-22T12:33:51,482 DEBUG [RegionOpenAndInit-ns2:test-16900290656251-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16900290656251,,1690029231061.01e992ceb654b1e583b990da186e85a8. 2023-07-22T12:33:51,482 DEBUG [RegionOpenAndInit-ns2:test-16900290656251-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16900290656251,,1690029231061.01e992ceb654b1e583b990da186e85a8. after waiting 0 ms 2023-07-22T12:33:51,482 DEBUG [RegionOpenAndInit-ns2:test-16900290656251-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16900290656251,,1690029231061.01e992ceb654b1e583b990da186e85a8. 2023-07-22T12:33:51,482 INFO [RegionOpenAndInit-ns2:test-16900290656251-pool-0 {}] regionserver.HRegion(1909): Closed ns2:test-16900290656251,,1690029231061.01e992ceb654b1e583b990da186e85a8. 2023-07-22T12:33:51,482 DEBUG [RegionOpenAndInit-ns2:test-16900290656251-pool-0 {}] regionserver.HRegion(1622): Region close journal for 01e992ceb654b1e583b990da186e85a8: 2023-07-22T12:33:51,483 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:test-16900290656251,,1690029231061.01e992ceb654b1e583b990da186e85a8.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1690029231483"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690029231483"}]},"ts":"1690029231483"} 2023-07-22T12:33:51,484 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:33:51,485 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16900290656251","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029231485"}]},"ts":"1690029231485"} 2023-07-22T12:33:51,486 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16900290656251, state=ENABLING in hbase:meta 2023-07-22T12:33:51,491 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16900290656251, region=01e992ceb654b1e583b990da186e85a8, ASSIGN}] 2023-07-22T12:33:51,492 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16900290656251, region=01e992ceb654b1e583b990da186e85a8, ASSIGN 2023-07-22T12:33:51,492 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:test-16900290656251, region=01e992ceb654b1e583b990da186e85a8, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,43639,1690029056327; forceNewPlan=false, retain=false 2023-07-22T12:33:51,644 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=125 updating hbase:meta row=01e992ceb654b1e583b990da186e85a8, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:33:51,645 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE; OpenRegionProcedure 01e992ceb654b1e583b990da186e85a8, server=jenkins-hbase3.apache.org,43639,1690029056327}] 2023-07-22T12:33:51,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=124 2023-07-22T12:33:51,800 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(130): Open ns2:test-16900290656251,,1690029231061.01e992ceb654b1e583b990da186e85a8. 2023-07-22T12:33:51,800 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7254): Opening region: {ENCODED => 01e992ceb654b1e583b990da186e85a8, NAME => 'ns2:test-16900290656251,,1690029231061.01e992ceb654b1e583b990da186e85a8.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:33:51,800 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:33:51,801 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16900290656251 01e992ceb654b1e583b990da186e85a8 2023-07-22T12:33:51,801 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(888): Instantiated ns2:test-16900290656251,,1690029231061.01e992ceb654b1e583b990da186e85a8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:33:51,801 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7296): checking encryption for 01e992ceb654b1e583b990da186e85a8 2023-07-22T12:33:51,801 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7299): checking classloading for 01e992ceb654b1e583b990da186e85a8 2023-07-22T12:33:51,802 INFO [StoreOpener-01e992ceb654b1e583b990da186e85a8-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 01e992ceb654b1e583b990da186e85a8 2023-07-22T12:33:51,803 INFO [StoreOpener-01e992ceb654b1e583b990da186e85a8-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 01e992ceb654b1e583b990da186e85a8 columnFamilyName f 2023-07-22T12:33:51,803 DEBUG [StoreOpener-01e992ceb654b1e583b990da186e85a8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:33:51,803 INFO [StoreOpener-01e992ceb654b1e583b990da186e85a8-1 {}] regionserver.HStore(324): Store=01e992ceb654b1e583b990da186e85a8/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:33:51,804 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/01e992ceb654b1e583b990da186e85a8 2023-07-22T12:33:51,804 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/01e992ceb654b1e583b990da186e85a8 2023-07-22T12:33:51,806 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1076): writing seq id for 01e992ceb654b1e583b990da186e85a8 2023-07-22T12:33:51,807 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/01e992ceb654b1e583b990da186e85a8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:33:51,808 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1093): Opened 01e992ceb654b1e583b990da186e85a8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9463684320, jitterRate=-0.1186257153749466}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-07-22T12:33:51,808 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(992): Region open journal for 01e992ceb654b1e583b990da186e85a8: 2023-07-22T12:33:51,809 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2552): Post open deploy tasks for ns2:test-16900290656251,,1690029231061.01e992ceb654b1e583b990da186e85a8., pid=126, masterSystemTime=1690029231797 2023-07-22T12:33:51,810 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2579): Finished post open deploy task for ns2:test-16900290656251,,1690029231061.01e992ceb654b1e583b990da186e85a8. 2023-07-22T12:33:51,810 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(158): Opened ns2:test-16900290656251,,1690029231061.01e992ceb654b1e583b990da186e85a8. 2023-07-22T12:33:51,810 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=125 updating hbase:meta row=01e992ceb654b1e583b990da186e85a8, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:33:51,813 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=126, resume processing ppid=125 2023-07-22T12:33:51,813 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=126, ppid=125, state=SUCCESS; OpenRegionProcedure 01e992ceb654b1e583b990da186e85a8, server=jenkins-hbase3.apache.org,43639,1690029056327 in 167 msec 2023-07-22T12:33:51,814 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=125, resume processing ppid=124 2023-07-22T12:33:51,814 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=125, ppid=124, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-16900290656251, region=01e992ceb654b1e583b990da186e85a8, ASSIGN in 322 msec 2023-07-22T12:33:51,814 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16900290656251","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029231814"}]},"ts":"1690029231814"} 2023-07-22T12:33:51,815 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16900290656251, state=ENABLED in hbase:meta 2023-07-22T12:33:51,818 DEBUG [PEWorker-4 {}] procedure.TruncateTableProcedure(145): truncate 'ns2:test-16900290656251' completed 2023-07-22T12:33:51,819 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=124, state=SUCCESS; TruncateTableProcedure (table=ns2:test-16900290656251 preserveSplits=true) in 771 msec 2023-07-22T12:33:52,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=124 2023-07-22T12:33:52,158 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:34745/backupUT/backup_1690029212049/ns2/test-16900290656251/archive/data/ns2/test-16900290656251/b8196e3466217e5359c3acacaaeb9e61 from hbase tables ns2:test-16900290656251 to tables ns2:test-16900290656251 2023-07-22T12:33:52,158 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceRestoreJob(72): Restore ns2:test-16900290656251 into ns2:test-16900290656251 2023-07-22T12:33:52,160 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-ns2-test-16900290656251-1690029232158 from hdfs://localhost:34745/backupUT/backup_1690029212049/ns2/test-16900290656251/archive/data/ns2/test-16900290656251/b8196e3466217e5359c3acacaaeb9e61 2023-07-22T12:33:52,165 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c54f61b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:33:52,166 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:33:52,167 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:37558, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:33:52,168 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-07-22T12:33:52,168 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:test-16900290656251 2023-07-22T12:33:52,171 DEBUG [hconnection-0x55d558f4-shared-pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:33:52,172 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:55594, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:33:52,173 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:test-16900290656251: 2023-07-22T12:33:52,173 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-07-22T12:33:52,173 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/partitions_c162db64-6371-410e-aaa5-2f9d39d0a27e 2023-07-22T12:33:52,579 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:52,580 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:52,580 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:52,580 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:53,572 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/hadoop-6307897033058637789.jar 2023-07-22T12:33:53,572 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:53,573 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:53,573 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:53,574 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:53,574 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:53,574 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:53,575 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:53,575 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:53,575 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:53,576 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:33:53,576 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-07-22T12:33:53,577 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-07-22T12:33:53,577 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-07-22T12:33:53,577 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-07-22T12:33:53,578 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-07-22T12:33:53,578 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-07-22T12:33:53,578 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-07-22T12:33:53,579 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-07-22T12:33:53,579 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-07-22T12:33:53,580 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-07-22T12:33:53,580 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-07-22T12:33:53,580 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:53,581 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:53,581 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:53,581 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:53,582 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:53,582 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:33:53,583 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:33:53,583 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: ns2:test-16900290656251 2023-07-22T12:33:53,583 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:33:53,583 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-07-22T12:33:53,583 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-07-22T12:33:53,584 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:33:53,594 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-07-22T12:33:53,669 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029233634/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-07-22T12:33:53,669 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029233634/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-07-22T12:33:53,720 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:34745/backupUT/backup_1690029212049/ns2/test-16900290656251/archive/data/ns2/test-16900290656251/b8196e3466217e5359c3acacaaeb9e61/f/82851155b50f47edb054b7e8d9f06090_SeqId_4_ 2023-07-22T12:33:53,723 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-07-22T12:33:53,725 DEBUG [LocalJobRunner Map Task Executor #0 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 82851155b50f47edb054b7e8d9f06090_SeqId_4_ 2023-07-22T12:33:53,745 DEBUG [pool-438-thread-1 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72b4689a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:33:53,746 DEBUG [hconnection-0x72edac35-metaLookup-shared--pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:33:53,748 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:55610, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:33:53,752 DEBUG [pool-438-thread-1 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:33:53,752 DEBUG [pool-438-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-07-22T12:33:53,752 DEBUG [pool-438-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-07-22T12:33:53,758 DEBUG [pool-438-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-ns2-test-16900290656251-1690029232158/_temporary/0/_temporary/attempt_local805605994_0019_r_000000_0/f/f4d88a34ceb648d2894885a36737c35d is 37, key is row10/f:q1/1690029070502/Put/seqid=0 2023-07-22T12:33:53,759 WARN [Thread-2091 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:43639] Specified favored nodes: [jenkins-hbase3.apache.org:43639] 2023-07-22T12:33:54,721 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-ns2-test-16900290656251-1690029232158 2023-07-22T12:33:54,725 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6848d223, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:33:54,727 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:33:54,728 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:55612, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:33:54,733 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-ns2-test-16900290656251-1690029232158/_SUCCESS 2023-07-22T12:33:54,736 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:33:54,738 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:37574, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:33:54,746 DEBUG [LoadIncrementalHFiles-0 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: f4d88a34ceb648d2894885a36737c35d 2023-07-22T12:33:54,746 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-ns2-test-16900290656251-1690029232158/f/f4d88a34ceb648d2894885a36737c35d first=Optional[row0] last=Optional[row98] 2023-07-22T12:33:54,747 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=ns2:test-16900290656251,,1690029231061.01e992ceb654b1e583b990da186e85a8., hostname=jenkins-hbase3.apache.org,43639,1690029056327, seqNum=2 for row with hfile group [{f,hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-ns2-test-16900290656251-1690029232158/f/f4d88a34ceb648d2894885a36737c35d}] 2023-07-22T12:33:54,752 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-ns2-test-16900290656251-1690029232158/f/f4d88a34ceb648d2894885a36737c35d for inclusion in 01e992ceb654b1e583b990da186e85a8/f 2023-07-22T12:33:54,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-07-22T12:33:54,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HStore(625): Region bounds: first= last= 2023-07-22T12:33:54,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: f4d88a34ceb648d2894885a36737c35d 2023-07-22T12:33:54,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HRegion(2520): Flush status journal for 01e992ceb654b1e583b990da186e85a8: 2023-07-22T12:33:54,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-ns2-test-16900290656251-1690029232158/f/f4d88a34ceb648d2894885a36737c35d to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/staging/jenkins__ns2_test-16900290656251__aam3tkcnloec31dfnaort0gf5mgd06g4iojb9g2reeik83b3jiuhhhlsng9k8lvb/f/f4d88a34ceb648d2894885a36737c35d 2023-07-22T12:33:54,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/staging/jenkins__ns2_test-16900290656251__aam3tkcnloec31dfnaort0gf5mgd06g4iojb9g2reeik83b3jiuhhhlsng9k8lvb/f/f4d88a34ceb648d2894885a36737c35d as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/01e992ceb654b1e583b990da186e85a8/f/d63801d92d8645ef95d1060b6ba3bbd9_SeqId_4_ 2023-07-22T12:33:54,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18b815d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:33:54,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:33:54,764 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:37578, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:33:54,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:33:54,768 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:55620, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:33:54,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] impl.BackupSystemTable(408): write bulk load descriptor to backup ns2:test-16900290656251 with 1 entries 2023-07-22T12:33:54,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] impl.BackupSystemTable(1582): writing raw bulk path hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/01e992ceb654b1e583b990da186e85a8/f/d63801d92d8645ef95d1060b6ba3bbd9_SeqId_4_ for ns2:test-16900290656251 01e992ceb654b1e583b990da186e85a8 2023-07-22T12:33:54,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] impl.BackupSystemTable(415): written 1 rows for bulk load of ns2:test-16900290656251 2023-07-22T12:33:54,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:33:54,781 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/staging/jenkins__ns2_test-16900290656251__aam3tkcnloec31dfnaort0gf5mgd06g4iojb9g2reeik83b3jiuhhhlsng9k8lvb/f/f4d88a34ceb648d2894885a36737c35d into 01e992ceb654b1e583b990da186e85a8/f as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/01e992ceb654b1e583b990da186e85a8/f/d63801d92d8645ef95d1060b6ba3bbd9_SeqId_4_ - updating store file list. 2023-07-22T12:33:54,785 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/01e992ceb654b1e583b990da186e85a8/f/d63801d92d8645ef95d1060b6ba3bbd9_SeqId_4_ into 01e992ceb654b1e583b990da186e85a8/f 2023-07-22T12:33:54,785 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/staging/jenkins__ns2_test-16900290656251__aam3tkcnloec31dfnaort0gf5mgd06g4iojb9g2reeik83b3jiuhhhlsng9k8lvb/f/f4d88a34ceb648d2894885a36737c35d into 01e992ceb654b1e583b990da186e85a8/f (new location: hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/01e992ceb654b1e583b990da186e85a8/f/d63801d92d8645ef95d1060b6ba3bbd9_SeqId_4_) 2023-07-22T12:33:54,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/staging/jenkins__ns2_test-16900290656251__aam3tkcnloec31dfnaort0gf5mgd06g4iojb9g2reeik83b3jiuhhhlsng9k8lvb/f/f4d88a34ceb648d2894885a36737c35d 2023-07-22T12:33:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e27d2ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:33:54,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:33:54,794 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:37584, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-07-22T12:33:54,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:33:54,797 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:55626, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-07-22T12:33:54,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] impl.BackupSystemTable(388): write bulk load descriptor to backup ns2:test-16900290656251 with 1 entries 2023-07-22T12:33:54,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] impl.BackupSystemTable(1503): writing done bulk path hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/01e992ceb654b1e583b990da186e85a8/f/d63801d92d8645ef95d1060b6ba3bbd9_SeqId_4_ for ns2:test-16900290656251 01e992ceb654b1e583b990da186e85a8 2023-07-22T12:33:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] impl.BackupSystemTable(394): written 1 rows for bulk load of ns2:test-16900290656251 2023-07-22T12:33:54,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:33:54,814 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:33:54,814 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-07-22T12:33:54,814 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-07-22T12:33:54,814 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(233): Backup: backup_1690029212049 hdfs://localhost:34745/backupUT/backup_1690029212049/ns2/test-16900290656251/ 2023-07-22T12:33:54,815 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34745/backupUT/backup_1690029212049/.backup.manifest 2023-07-22T12:33:54,817 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029212049/.backup.manifest 2023-07-22T12:33:54,817 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(148): Restoring 'ns3:test-16900290656252' to 'ns3:test-16900290656252' from full backup image hdfs://localhost:34745/backupUT/backup_1690029212049/ns3/test-16900290656252 2023-07-22T12:33:54,822 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] util.RestoreTool(101): Folder tableArchivePath: hdfs://localhost:34745/backupUT/backup_1690029212049/ns3/test-16900290656252/archive/data/ns3/test-16900290656252 does not exists 2023-07-22T12:33:54,822 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] util.RestoreTool(326): find table descriptor but no archive dir for table ns3:test-16900290656252, will only create table 2023-07-22T12:33:54,822 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] util.RestoreTool(476): Truncating exising target table 'ns3:test-16900290656252', preserving region splits 2023-07-22T12:33:54,823 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster$12(2663): Client=jenkins//172.31.12.81 disable ns3:test-16900290656252 2023-07-22T12:33:54,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=127, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=ns3:test-16900290656252 2023-07-22T12:33:54,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=127 2023-07-22T12:33:54,826 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16900290656252","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029234825"}]},"ts":"1690029234825"} 2023-07-22T12:33:54,827 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16900290656252, state=DISABLING in hbase:meta 2023-07-22T12:33:54,830 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(293): Set ns3:test-16900290656252 to state=DISABLING 2023-07-22T12:33:54,830 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns3:test-16900290656252, region=aefea80783ea312734eb5d79ad00b2f1, UNASSIGN}] 2023-07-22T12:33:54,831 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=128, ppid=127, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns3:test-16900290656252, region=aefea80783ea312734eb5d79ad00b2f1, UNASSIGN 2023-07-22T12:33:54,832 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=128 updating hbase:meta row=aefea80783ea312734eb5d79ad00b2f1, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:33:54,833 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=129, ppid=128, state=RUNNABLE; CloseRegionProcedure aefea80783ea312734eb5d79ad00b2f1, server=jenkins-hbase3.apache.org,43639,1690029056327}] 2023-07-22T12:33:54,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=127 2023-07-22T12:33:54,984 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] handler.UnassignRegionHandler(119): Close aefea80783ea312734eb5d79ad00b2f1 2023-07-22T12:33:54,984 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(1668): Closing aefea80783ea312734eb5d79ad00b2f1, disabling compactions & flushes 2023-07-22T12:33:54,984 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(1690): Closing region ns3:test-16900290656252,,1690029100393.aefea80783ea312734eb5d79ad00b2f1. 2023-07-22T12:33:54,985 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16900290656252,,1690029100393.aefea80783ea312734eb5d79ad00b2f1. 2023-07-22T12:33:54,985 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16900290656252,,1690029100393.aefea80783ea312734eb5d79ad00b2f1. after waiting 0 ms 2023-07-22T12:33:54,985 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16900290656252,,1690029100393.aefea80783ea312734eb5d79ad00b2f1. 2023-07-22T12:33:54,988 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns3/test-16900290656252/aefea80783ea312734eb5d79ad00b2f1/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=1 2023-07-22T12:33:54,988 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:33:54,989 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(1909): Closed ns3:test-16900290656252,,1690029100393.aefea80783ea312734eb5d79ad00b2f1. 2023-07-22T12:33:54,989 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(1622): Region close journal for aefea80783ea312734eb5d79ad00b2f1: 2023-07-22T12:33:54,990 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] handler.UnassignRegionHandler(163): Closed aefea80783ea312734eb5d79ad00b2f1 2023-07-22T12:33:54,991 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=128 updating hbase:meta row=aefea80783ea312734eb5d79ad00b2f1, regionState=CLOSED 2023-07-22T12:33:54,993 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=129, resume processing ppid=128 2023-07-22T12:33:54,993 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=129, ppid=128, state=SUCCESS; CloseRegionProcedure aefea80783ea312734eb5d79ad00b2f1, server=jenkins-hbase3.apache.org,43639,1690029056327 in 159 msec 2023-07-22T12:33:54,994 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=128, resume processing ppid=127 2023-07-22T12:33:54,994 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=128, ppid=127, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-16900290656252, region=aefea80783ea312734eb5d79ad00b2f1, UNASSIGN in 163 msec 2023-07-22T12:33:54,994 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16900290656252","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029234994"}]},"ts":"1690029234994"} 2023-07-22T12:33:54,995 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16900290656252, state=DISABLED in hbase:meta 2023-07-22T12:33:54,997 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(305): Set ns3:test-16900290656252 to state=DISABLED 2023-07-22T12:33:54,998 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=127, state=SUCCESS; DisableTableProcedure table=ns3:test-16900290656252 in 174 msec 2023-07-22T12:33:55,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=127 2023-07-22T12:33:55,128 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster$6(2479): Client=jenkins//172.31.12.81 truncate ns3:test-16900290656252 2023-07-22T12:33:55,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=130, state=RUNNABLE:TRUNCATE_TABLE_PRE_OPERATION; TruncateTableProcedure (table=ns3:test-16900290656252 preserveSplits=true) 2023-07-22T12:33:55,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=130 2023-07-22T12:33:55,130 DEBUG [PEWorker-1 {}] procedure.TruncateTableProcedure(87): waiting for 'ns3:test-16900290656252' regions in transition 2023-07-22T12:33:55,132 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns3/test-16900290656252/aefea80783ea312734eb5d79ad00b2f1 2023-07-22T12:33:55,133 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns3/test-16900290656252/aefea80783ea312734eb5d79ad00b2f1/f, FileablePath, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns3/test-16900290656252/aefea80783ea312734eb5d79ad00b2f1/recovered.edits] 2023-07-22T12:33:55,137 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns3/test-16900290656252/aefea80783ea312734eb5d79ad00b2f1/recovered.edits/7.seqid to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/archive/data/ns3/test-16900290656252/aefea80783ea312734eb5d79ad00b2f1/recovered.edits/7.seqid 2023-07-22T12:33:55,138 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns3/test-16900290656252/aefea80783ea312734eb5d79ad00b2f1 2023-07-22T12:33:55,138 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(293): Archived ns3:test-16900290656252 regions 2023-07-22T12:33:55,141 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of ns3:test-16900290656252 from hbase:meta 2023-07-22T12:33:55,143 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(387): Removing 'ns3:test-16900290656252' descriptor. 2023-07-22T12:33:55,143 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(377): Removing 'ns3:test-16900290656252' from region states. 2023-07-22T12:33:55,143 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns3:test-16900290656252,,1690029100393.aefea80783ea312734eb5d79ad00b2f1.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1690029235143"}]},"ts":"9223372036854775807"} 2023-07-22T12:33:55,145 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-07-22T12:33:55,145 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => aefea80783ea312734eb5d79ad00b2f1, NAME => 'ns3:test-16900290656252,,1690029100393.aefea80783ea312734eb5d79ad00b2f1.', STARTKEY => '', ENDKEY => ''}] 2023-07-22T12:33:55,145 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(381): Marking 'ns3:test-16900290656252' as deleted. 2023-07-22T12:33:55,145 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns3:test-16900290656252","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1690029235145"}]},"ts":"9223372036854775807"} 2023-07-22T12:33:55,148 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1678): Deleted table ns3:test-16900290656252 state from META 2023-07-22T12:33:55,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=130 2023-07-22T12:33:55,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=130 2023-07-22T12:33:55,557 INFO [RegionOpenAndInit-ns3:test-16900290656252-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 7e12e84cd2aa3dccee1c81fd5a1ead69, NAME => 'ns3:test-16900290656252,,1690029235139.7e12e84cd2aa3dccee1c81fd5a1ead69.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-16900290656252', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947 2023-07-22T12:33:55,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=130 2023-07-22T12:33:55,963 DEBUG [RegionOpenAndInit-ns3:test-16900290656252-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:test-16900290656252,,1690029235139.7e12e84cd2aa3dccee1c81fd5a1ead69.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:33:55,963 DEBUG [RegionOpenAndInit-ns3:test-16900290656252-pool-0 {}] regionserver.HRegion(1668): Closing 7e12e84cd2aa3dccee1c81fd5a1ead69, disabling compactions & flushes 2023-07-22T12:33:55,963 INFO [RegionOpenAndInit-ns3:test-16900290656252-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:test-16900290656252,,1690029235139.7e12e84cd2aa3dccee1c81fd5a1ead69. 2023-07-22T12:33:55,963 DEBUG [RegionOpenAndInit-ns3:test-16900290656252-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16900290656252,,1690029235139.7e12e84cd2aa3dccee1c81fd5a1ead69. 2023-07-22T12:33:55,963 DEBUG [RegionOpenAndInit-ns3:test-16900290656252-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16900290656252,,1690029235139.7e12e84cd2aa3dccee1c81fd5a1ead69. after waiting 0 ms 2023-07-22T12:33:55,963 DEBUG [RegionOpenAndInit-ns3:test-16900290656252-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16900290656252,,1690029235139.7e12e84cd2aa3dccee1c81fd5a1ead69. 2023-07-22T12:33:55,963 INFO [RegionOpenAndInit-ns3:test-16900290656252-pool-0 {}] regionserver.HRegion(1909): Closed ns3:test-16900290656252,,1690029235139.7e12e84cd2aa3dccee1c81fd5a1ead69. 2023-07-22T12:33:55,963 DEBUG [RegionOpenAndInit-ns3:test-16900290656252-pool-0 {}] regionserver.HRegion(1622): Region close journal for 7e12e84cd2aa3dccee1c81fd5a1ead69: 2023-07-22T12:33:55,964 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:test-16900290656252,,1690029235139.7e12e84cd2aa3dccee1c81fd5a1ead69.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1690029235964"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690029235964"}]},"ts":"1690029235964"} 2023-07-22T12:33:55,966 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:33:55,966 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16900290656252","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029235966"}]},"ts":"1690029235966"} 2023-07-22T12:33:55,967 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16900290656252, state=ENABLING in hbase:meta 2023-07-22T12:33:55,971 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16900290656252, region=7e12e84cd2aa3dccee1c81fd5a1ead69, ASSIGN}] 2023-07-22T12:33:55,973 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16900290656252, region=7e12e84cd2aa3dccee1c81fd5a1ead69, ASSIGN 2023-07-22T12:33:55,973 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:test-16900290656252, region=7e12e84cd2aa3dccee1c81fd5a1ead69, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,43639,1690029056327; forceNewPlan=false, retain=false 2023-07-22T12:33:56,124 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=131 updating hbase:meta row=7e12e84cd2aa3dccee1c81fd5a1ead69, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:33:56,126 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE; OpenRegionProcedure 7e12e84cd2aa3dccee1c81fd5a1ead69, server=jenkins-hbase3.apache.org,43639,1690029056327}] 2023-07-22T12:33:56,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=130 2023-07-22T12:33:56,280 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] handler.AssignRegionHandler(130): Open ns3:test-16900290656252,,1690029235139.7e12e84cd2aa3dccee1c81fd5a1ead69. 2023-07-22T12:33:56,280 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(7254): Opening region: {ENCODED => 7e12e84cd2aa3dccee1c81fd5a1ead69, NAME => 'ns3:test-16900290656252,,1690029235139.7e12e84cd2aa3dccee1c81fd5a1ead69.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:33:56,280 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:33:56,280 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16900290656252 7e12e84cd2aa3dccee1c81fd5a1ead69 2023-07-22T12:33:56,280 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(888): Instantiated ns3:test-16900290656252,,1690029235139.7e12e84cd2aa3dccee1c81fd5a1ead69.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:33:56,280 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(7296): checking encryption for 7e12e84cd2aa3dccee1c81fd5a1ead69 2023-07-22T12:33:56,280 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(7299): checking classloading for 7e12e84cd2aa3dccee1c81fd5a1ead69 2023-07-22T12:33:56,281 INFO [StoreOpener-7e12e84cd2aa3dccee1c81fd5a1ead69-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 7e12e84cd2aa3dccee1c81fd5a1ead69 2023-07-22T12:33:56,283 INFO [StoreOpener-7e12e84cd2aa3dccee1c81fd5a1ead69-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7e12e84cd2aa3dccee1c81fd5a1ead69 columnFamilyName f 2023-07-22T12:33:56,283 DEBUG [StoreOpener-7e12e84cd2aa3dccee1c81fd5a1ead69-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:33:56,283 INFO [StoreOpener-7e12e84cd2aa3dccee1c81fd5a1ead69-1 {}] regionserver.HStore(324): Store=7e12e84cd2aa3dccee1c81fd5a1ead69/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:33:56,284 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns3/test-16900290656252/7e12e84cd2aa3dccee1c81fd5a1ead69 2023-07-22T12:33:56,284 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns3/test-16900290656252/7e12e84cd2aa3dccee1c81fd5a1ead69 2023-07-22T12:33:56,285 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1076): writing seq id for 7e12e84cd2aa3dccee1c81fd5a1ead69 2023-07-22T12:33:56,287 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns3/test-16900290656252/7e12e84cd2aa3dccee1c81fd5a1ead69/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:33:56,287 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1093): Opened 7e12e84cd2aa3dccee1c81fd5a1ead69; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11899621600, jitterRate=0.10823862254619598}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-07-22T12:33:56,288 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(992): Region open journal for 7e12e84cd2aa3dccee1c81fd5a1ead69: 2023-07-22T12:33:56,288 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegionServer(2552): Post open deploy tasks for ns3:test-16900290656252,,1690029235139.7e12e84cd2aa3dccee1c81fd5a1ead69., pid=132, masterSystemTime=1690029236277 2023-07-22T12:33:56,289 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegionServer(2579): Finished post open deploy task for ns3:test-16900290656252,,1690029235139.7e12e84cd2aa3dccee1c81fd5a1ead69. 2023-07-22T12:33:56,289 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] handler.AssignRegionHandler(158): Opened ns3:test-16900290656252,,1690029235139.7e12e84cd2aa3dccee1c81fd5a1ead69. 2023-07-22T12:33:56,290 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=131 updating hbase:meta row=7e12e84cd2aa3dccee1c81fd5a1ead69, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:33:56,292 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=132, resume processing ppid=131 2023-07-22T12:33:56,292 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=132, ppid=131, state=SUCCESS; OpenRegionProcedure 7e12e84cd2aa3dccee1c81fd5a1ead69, server=jenkins-hbase3.apache.org,43639,1690029056327 in 165 msec 2023-07-22T12:33:56,293 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=131, resume processing ppid=130 2023-07-22T12:33:56,293 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=131, ppid=130, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-16900290656252, region=7e12e84cd2aa3dccee1c81fd5a1ead69, ASSIGN in 321 msec 2023-07-22T12:33:56,293 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16900290656252","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029236293"}]},"ts":"1690029236293"} 2023-07-22T12:33:56,294 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16900290656252, state=ENABLED in hbase:meta 2023-07-22T12:33:56,296 DEBUG [PEWorker-5 {}] procedure.TruncateTableProcedure(145): truncate 'ns3:test-16900290656252' completed 2023-07-22T12:33:56,297 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=130, state=SUCCESS; TruncateTableProcedure (table=ns3:test-16900290656252 preserveSplits=true) in 1.1680 sec 2023-07-22T12:33:56,458 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-07-22T12:33:57,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=130 2023-07-22T12:33:57,237 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-07-22T12:33:57,237 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(233): Backup: backup_1690029212049 hdfs://localhost:34745/backupUT/backup_1690029212049/ns3/test-16900290656252/ 2023-07-22T12:33:57,238 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-07-22T12:33:57,238 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:33:57,260 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreMultipleOverwriteCommand Thread=606 (was 601) - Thread LEAK? -, OpenFileDescriptor=686 (was 681) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=113 (was 154), ProcessCount=169 (was 169), AvailableMemoryMB=16393 (was 16983) 2023-07-22T12:33:57,260 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(130): Thread=606 is superior to 500 2023-07-22T12:33:57,275 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreMultipleCommand Thread=606, OpenFileDescriptor=686, MaxFileDescriptor=60000, SystemLoadAverage=113, ProcessCount=169, AvailableMemoryMB=16393 2023-07-22T12:33:57,275 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(130): Thread=606 is superior to 500 2023-07-22T12:33:57,275 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] backup.TestFullRestore(144): create full backup image on multiple tables: command-line 2023-07-22T12:33:57,280 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28e1cb39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:33:57,283 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:33:57,285 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:55634, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:33:57,287 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:33:57,288 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:37588, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:33:57,299 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-07-22T12:33:57,305 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:33:57,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029237305 (current time:1690029237305). 2023-07-22T12:33:57,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:33:57,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-07-22T12:33:57,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ca3eea8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:33:57,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:33:57,313 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:55646, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:33:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:33:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:33:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=133, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:33:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 133 2023-07-22T12:33:57,316 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:33:57,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=133 2023-07-22T12:33:57,317 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:33:57,318 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:33:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=133 2023-07-22T12:33:57,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=133 2023-07-22T12:33:57,724 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:33:57,724 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33}] 2023-07-22T12:33:57,726 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=134, ppid=133, state=RUNNABLE; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33 2023-07-22T12:33:57,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43639 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=134 2023-07-22T12:33:57,877 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. 2023-07-22T12:33:57,877 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.HRegion(2819): Flushing 697cc5ecf0962f5ab9f4126febf7ce33 2/2 column families, dataSize=1.50 KB heapSize=2.73 KB 2023-07-22T12:33:57,881 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/meta/e0ee030fbe0a48558b7d348517bf0f3c is 171, key is trslm:hdfs://localhost:34745/backupUT\x00ns2:test-16900290656251/meta:log-roll-map/1690029229874/Put/seqid=0 2023-07-22T12:33:57,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=133 2023-07-22T12:33:58,285 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=717 B at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/meta/e0ee030fbe0a48558b7d348517bf0f3c 2023-07-22T12:33:58,289 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: e0ee030fbe0a48558b7d348517bf0f3c 2023-07-22T12:33:58,292 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/session/82e877bb04384fa3a3f2266a337937ce is 524, key is session:backup_1690029212049/session:context/1690029230700/Put/seqid=0 2023-07-22T12:33:58,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=133 2023-07-22T12:33:58,696 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=820 B at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/session/82e877bb04384fa3a3f2266a337937ce 2023-07-22T12:33:58,700 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 82e877bb04384fa3a3f2266a337937ce 2023-07-22T12:33:58,701 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/meta/e0ee030fbe0a48558b7d348517bf0f3c as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/e0ee030fbe0a48558b7d348517bf0f3c 2023-07-22T12:33:58,705 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/e0ee030fbe0a48558b7d348517bf0f3c, entries=6, sequenceid=105, filesize=5.8 K 2023-07-22T12:33:58,706 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/session/82e877bb04384fa3a3f2266a337937ce as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/82e877bb04384fa3a3f2266a337937ce 2023-07-22T12:33:58,710 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/82e877bb04384fa3a3f2266a337937ce, entries=2, sequenceid=105, filesize=5.5 K 2023-07-22T12:33:58,711 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.HRegion(3022): Finished flush of dataSize ~1.50 KB/1537, heapSize ~2.70 KB/2760, currentSize=0 B/0 for 697cc5ecf0962f5ab9f4126febf7ce33 in 834ms, sequenceid=105, compaction requested=true 2023-07-22T12:33:58,711 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.HRegion(2520): Flush status journal for 697cc5ecf0962f5ab9f4126febf7ce33: 2023-07-22T12:33:58,711 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. for snapshot_backup_system completed. 2023-07-22T12:33:58,711 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33.' region-info for snapshot=snapshot_backup_system 2023-07-22T12:33:58,711 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:33:58,711 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/b580ac8bece04b07a21605e555bdc696, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/5ea0158e2d574f758c68ee9c20c21191, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/a6d2695c5574440ea495f58a7c66ed03, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/8f78c6cced064299a30d72a0b4901528, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/606f117b1e384a54bb6dba5339e90e8f, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/15bf4f9b1b65467c96713e751468dc6c, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/b30f291cd15a4d92ac4b31f477e93428, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/3e46ffe2abec4ceba3ab7424eff1a910, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/e0ee030fbe0a48558b7d348517bf0f3c] hfiles 2023-07-22T12:33:58,711 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (1/9): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/b580ac8bece04b07a21605e555bdc696 for snapshot=snapshot_backup_system 2023-07-22T12:33:58,711 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (2/9): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/5ea0158e2d574f758c68ee9c20c21191 for snapshot=snapshot_backup_system 2023-07-22T12:33:58,711 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (3/9): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/a6d2695c5574440ea495f58a7c66ed03 for snapshot=snapshot_backup_system 2023-07-22T12:33:58,711 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (4/9): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/8f78c6cced064299a30d72a0b4901528 for snapshot=snapshot_backup_system 2023-07-22T12:33:58,711 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (5/9): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/606f117b1e384a54bb6dba5339e90e8f for snapshot=snapshot_backup_system 2023-07-22T12:33:58,711 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (6/9): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/15bf4f9b1b65467c96713e751468dc6c for snapshot=snapshot_backup_system 2023-07-22T12:33:58,711 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (7/9): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/b30f291cd15a4d92ac4b31f477e93428 for snapshot=snapshot_backup_system 2023-07-22T12:33:58,711 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (8/9): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/3e46ffe2abec4ceba3ab7424eff1a910 for snapshot=snapshot_backup_system 2023-07-22T12:33:58,711 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (9/9): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/e0ee030fbe0a48558b7d348517bf0f3c for snapshot=snapshot_backup_system 2023-07-22T12:33:58,711 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/956010b3eaa64b4fa82ddea4557fac2c, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/0704967b71ce4a5cb0af553e6e47addb, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/62b5d2f1465b492ab802e26f5bf32f6b, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/c2fdeacbf4184719a11904adfeaf1fd8, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/660347589bb3465a82d9c9138151d6a4, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/2b4c40a0167b43f584d34520d18e9390, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/1550986ca81f443894a286cada73692c, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/a5d57e73d32d4eda8e52490df82775fa, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/83933de825724e92a18f2bfe93b821b6, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/82e877bb04384fa3a3f2266a337937ce] hfiles 2023-07-22T12:33:58,711 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (1/10): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/956010b3eaa64b4fa82ddea4557fac2c for snapshot=snapshot_backup_system 2023-07-22T12:33:58,711 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (2/10): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/0704967b71ce4a5cb0af553e6e47addb for snapshot=snapshot_backup_system 2023-07-22T12:33:58,711 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (3/10): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/62b5d2f1465b492ab802e26f5bf32f6b for snapshot=snapshot_backup_system 2023-07-22T12:33:58,711 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (4/10): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/c2fdeacbf4184719a11904adfeaf1fd8 for snapshot=snapshot_backup_system 2023-07-22T12:33:58,711 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (5/10): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/660347589bb3465a82d9c9138151d6a4 for snapshot=snapshot_backup_system 2023-07-22T12:33:58,711 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (6/10): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/2b4c40a0167b43f584d34520d18e9390 for snapshot=snapshot_backup_system 2023-07-22T12:33:58,712 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (7/10): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/1550986ca81f443894a286cada73692c for snapshot=snapshot_backup_system 2023-07-22T12:33:58,712 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (8/10): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/a5d57e73d32d4eda8e52490df82775fa for snapshot=snapshot_backup_system 2023-07-22T12:33:58,712 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (9/10): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/83933de825724e92a18f2bfe93b821b6 for snapshot=snapshot_backup_system 2023-07-22T12:33:58,712 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (10/10): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/82e877bb04384fa3a3f2266a337937ce for snapshot=snapshot_backup_system 2023-07-22T12:33:58,758 DEBUG [regionserver/jenkins-hbase3:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.5384615384615384, tune throughput to 76.92 MB/second 2023-07-22T12:33:59,119 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. 2023-07-22T12:33:59,119 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=134 2023-07-22T12:33:59,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster(3994): Remote procedure done, pid=134 2023-07-22T12:33:59,120 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 697cc5ecf0962f5ab9f4126febf7ce33 2023-07-22T12:33:59,121 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=134, ppid=133, state=RUNNABLE; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33 2023-07-22T12:33:59,123 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=134, resume processing ppid=133 2023-07-22T12:33:59,123 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:33:59,123 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=134, ppid=133, state=SUCCESS; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33 in 1.3980 sec 2023-07-22T12:33:59,123 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:33:59,124 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:33:59,124 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-07-22T12:33:59,124 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_backup_system 2023-07-22T12:33:59,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=133 2023-07-22T12:33:59,534 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:33:59,545 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:33:59,546 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_backup_system 2023-07-22T12:34:00,361 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:34:00,361 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 133 2023-07-22T12:34:00,362 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=133, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 3.0470 sec 2023-07-22T12:34:01,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=133 2023-07-22T12:34:01,422 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(115): Backup backup_1690029237280 started at 1690029241422. 2023-07-22T12:34:01,423 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(119): Backup session backup_1690029237280 has been started. 2023-07-22T12:34:01,428 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-07-22T12:34:01,429 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-07-22T12:34:01,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-07-22T12:34:01,429 INFO [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-07-22T12:34:01,429 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-07-22T12:34:01,430 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-07-22T12:34:01,430 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-07-22T12:34:01,431 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:34:01,433 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:01,433 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:34:01,433 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:34:01,433 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-07-22T12:34:01,433 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-07-22T12:34:01,434 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-07-22T12:34:01,434 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:34:01,434 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-07-22T12:34:01,434 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-07-22T12:34:01,434 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-07-22T12:34:01,434 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-07-22T12:34:01,434 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-07-22T12:34:01,437 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-07-22T12:34:01,437 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-07-22T12:34:01,437 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,43639,1690029056327' joining acquired barrier for procedure (rolllog) in zk 2023-07-22T12:34:01,441 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-07-22T12:34:01,441 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:01,441 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:34:01,441 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:34:01,441 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-07-22T12:34:01,441 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:34:01,442 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:34:01,442 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:34:01,442 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:01,442 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:34:01,443 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,43639,1690029056327' joining acquired barrier for procedure 'rolllog' on coordinator 2023-07-22T12:34:01,443 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@58c5aa33[Count = 0] remaining members to acquire global barrier 2023-07-22T12:34:01,443 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-07-22T12:34:01,443 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-07-22T12:34:01,444 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-07-22T12:34:01,444 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-07-22T12:34:01,444 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-07-22T12:34:01,444 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-07-22T12:34:01,445 DEBUG [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:01,445 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-07-22T12:34:01,445 INFO [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1690029216231 highest: 1690029216231 on jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:01,445 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta:.meta(num 1690029216214) roll requested 2023-07-22T12:34:01,456 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029241445.meta, exclude list is [], retry=0 2023-07-22T12:34:01,459 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK] 2023-07-22T12:34:01,461 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029216214.meta with entries=24, filesize=6.28 KB; new WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029241445.meta 2023-07-22T12:34:01,461 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK]] 2023-07-22T12:34:01,461 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029216214.meta is not closed yet, will try archiving it next time 2023-07-22T12:34:01,461 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C43639%2C1690029056327:(num 1690029216231) roll requested 2023-07-22T12:34:01,473 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029241461, exclude list is [], retry=0 2023-07-22T12:34:01,476 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK] 2023-07-22T12:34:01,477 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029216231 with entries=17, filesize=5.81 KB; new WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029241461 2023-07-22T12:34:01,477 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK]] 2023-07-22T12:34:01,478 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029216231 is not closed yet, will try archiving it next time 2023-07-22T12:34:01,478 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029191975 to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/oldWALs/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029191975 2023-07-22T12:34:01,481 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029216231 not finished, retry = 0 2023-07-22T12:34:01,485 DEBUG [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-07-22T12:34:01,485 INFO [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1690029241461 on jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:01,492 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-07-22T12:34:01,492 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,43639,1690029056327' in zk 2023-07-22T12:34:01,495 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-07-22T12:34:01,495 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:01,495 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:34:01,495 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:34:01,495 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-07-22T12:34:01,495 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:34:01,496 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:34:01,496 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:34:01,497 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:34:01,497 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:01,497 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:34:01,497 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:34:01,498 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:01,498 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,43639,1690029056327': 2023-07-22T12:34:01,498 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,43639,1690029056327' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-07-22T12:34:01,498 INFO [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-07-22T12:34:01,498 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-07-22T12:34:01,498 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-07-22T12:34:01,498 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-07-22T12:34:01,498 INFO [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-07-22T12:34:01,501 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-07-22T12:34:01,501 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-07-22T12:34:01,501 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:34:01,501 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:34:01,501 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:34:01,502 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:34:01,502 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:34:01,502 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:34:01,502 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:34:01,502 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:34:01,503 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:34:01,503 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:34:01,503 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:01,504 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:34:01,504 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:34:01,504 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:01,511 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:34:01,511 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:34:01,511 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-07-22T12:34:01,511 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-07-22T12:34:01,511 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:34:01,512 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:34:01,512 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:34:01,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-07-22T12:34:01,616 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1690029241615_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } 2023-07-22T12:34:01,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029241616 (current time:1690029241616). 2023-07-22T12:34:01,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:34:01,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1690029241615_ns2_test-16900290656251 VERSION not specified, setting to 2 2023-07-22T12:34:01,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47f54475, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:34:01,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:34:01,626 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:34310, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:34:01,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:34:01,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:34:01,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=135, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1690029241615_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } 2023-07-22T12:34:01,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1690029241615_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 }, snapshot procedure id = 135 2023-07-22T12:34:01,630 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1690029241615_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:34:01,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=135 2023-07-22T12:34:01,631 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1690029241615_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:34:01,633 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1690029241615_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:34:01,638 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1690029241615_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:34:01,638 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE; SnapshotRegionProcedure 01e992ceb654b1e583b990da186e85a8}] 2023-07-22T12:34:01,640 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=136, ppid=135, state=RUNNABLE; SnapshotRegionProcedure 01e992ceb654b1e583b990da186e85a8 2023-07-22T12:34:01,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=135 2023-07-22T12:34:01,782 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-07-22T12:34:01,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43639 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=136 2023-07-22T12:34:01,791 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-16900290656251,,1690029231061.01e992ceb654b1e583b990da186e85a8. 2023-07-22T12:34:01,791 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] regionserver.HRegion(2520): Flush status journal for 01e992ceb654b1e583b990da186e85a8: 2023-07-22T12:34:01,792 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-16900290656251,,1690029231061.01e992ceb654b1e583b990da186e85a8. for snapshot_1690029241615_ns2_test-16900290656251 completed. 2023-07-22T12:34:01,792 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] snapshot.SnapshotManifest(240): Storing 'ns2:test-16900290656251,,1690029231061.01e992ceb654b1e583b990da186e85a8.' region-info for snapshot=snapshot_1690029241615_ns2_test-16900290656251 2023-07-22T12:34:01,792 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:34:01,792 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/01e992ceb654b1e583b990da186e85a8/f/d63801d92d8645ef95d1060b6ba3bbd9_SeqId_4_] hfiles 2023-07-22T12:34:01,792 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/01e992ceb654b1e583b990da186e85a8/f/d63801d92d8645ef95d1060b6ba3bbd9_SeqId_4_ for snapshot=snapshot_1690029241615_ns2_test-16900290656251 2023-07-22T12:34:01,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=135 2023-07-22T12:34:02,199 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-16900290656251,,1690029231061.01e992ceb654b1e583b990da186e85a8. 2023-07-22T12:34:02,199 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=136 2023-07-22T12:34:02,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster(3994): Remote procedure done, pid=136 2023-07-22T12:34:02,200 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1690029241615_ns2_test-16900290656251 on region 01e992ceb654b1e583b990da186e85a8 2023-07-22T12:34:02,201 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=136, ppid=135, state=RUNNABLE; SnapshotRegionProcedure 01e992ceb654b1e583b990da186e85a8 2023-07-22T12:34:02,203 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=136, resume processing ppid=135 2023-07-22T12:34:02,203 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=136, ppid=135, state=SUCCESS; SnapshotRegionProcedure 01e992ceb654b1e583b990da186e85a8 in 564 msec 2023-07-22T12:34:02,203 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1690029241615_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:34:02,203 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1690029241615_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:34:02,204 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1690029241615_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:34:02,204 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1690029241615_ns2_test-16900290656251 2023-07-22T12:34:02,204 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_1690029241615_ns2_test-16900290656251 2023-07-22T12:34:02,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=135 2023-07-22T12:34:02,613 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1690029241615_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:34:02,618 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1690029241615_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:34:02,619 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_1690029241615_ns2_test-16900290656251 to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029241615_ns2_test-16900290656251 2023-07-22T12:34:02,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=135 2023-07-22T12:34:03,434 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1690029241615_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:34:03,434 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1690029241615_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 }, snapshot procedure id = 135 2023-07-22T12:34:03,435 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=135, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1690029241615_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } in 1.8060 sec 2023-07-22T12:34:03,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=135 2023-07-22T12:34:03,736 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1690029243735_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } 2023-07-22T12:34:03,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029243736 (current time:1690029243736). 2023-07-22T12:34:03,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:34:03,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1690029243735_ns3_test-16900290656252 VERSION not specified, setting to 2 2023-07-22T12:34:03,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b58e58b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:34:03,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:34:03,745 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:34322, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:34:03,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:34:03,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:34:03,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=137, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1690029243735_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } 2023-07-22T12:34:03,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1690029243735_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 }, snapshot procedure id = 137 2023-07-22T12:34:03,748 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1690029243735_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:34:03,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=137 2023-07-22T12:34:03,749 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1690029243735_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:34:03,750 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1690029243735_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:34:03,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=137 2023-07-22T12:34:04,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=137 2023-07-22T12:34:04,156 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1690029243735_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:34:04,156 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE; SnapshotRegionProcedure 7e12e84cd2aa3dccee1c81fd5a1ead69}] 2023-07-22T12:34:04,158 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=138, ppid=137, state=RUNNABLE; SnapshotRegionProcedure 7e12e84cd2aa3dccee1c81fd5a1ead69 2023-07-22T12:34:04,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43639 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=138 2023-07-22T12:34:04,309 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=138}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-16900290656252,,1690029235139.7e12e84cd2aa3dccee1c81fd5a1ead69. 2023-07-22T12:34:04,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=138}] regionserver.HRegion(2520): Flush status journal for 7e12e84cd2aa3dccee1c81fd5a1ead69: 2023-07-22T12:34:04,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=138}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-16900290656252,,1690029235139.7e12e84cd2aa3dccee1c81fd5a1ead69. for snapshot_1690029243735_ns3_test-16900290656252 completed. 2023-07-22T12:34:04,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=138}] snapshot.SnapshotManifest(240): Storing 'ns3:test-16900290656252,,1690029235139.7e12e84cd2aa3dccee1c81fd5a1ead69.' region-info for snapshot=snapshot_1690029243735_ns3_test-16900290656252 2023-07-22T12:34:04,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=138}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:34:04,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=138}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-07-22T12:34:04,316 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=138}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-16900290656252,,1690029235139.7e12e84cd2aa3dccee1c81fd5a1ead69. 2023-07-22T12:34:04,316 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=138 2023-07-22T12:34:04,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster(3994): Remote procedure done, pid=138 2023-07-22T12:34:04,316 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1690029243735_ns3_test-16900290656252 on region 7e12e84cd2aa3dccee1c81fd5a1ead69 2023-07-22T12:34:04,317 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=138, ppid=137, state=RUNNABLE; SnapshotRegionProcedure 7e12e84cd2aa3dccee1c81fd5a1ead69 2023-07-22T12:34:04,319 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=138, resume processing ppid=137 2023-07-22T12:34:04,319 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1690029243735_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:34:04,319 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=138, ppid=137, state=SUCCESS; SnapshotRegionProcedure 7e12e84cd2aa3dccee1c81fd5a1ead69 in 162 msec 2023-07-22T12:34:04,320 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1690029243735_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:34:04,320 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1690029243735_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:34:04,320 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1690029243735_ns3_test-16900290656252 2023-07-22T12:34:04,321 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_1690029243735_ns3_test-16900290656252 2023-07-22T12:34:04,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=137 2023-07-22T12:34:04,729 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1690029243735_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:34:04,733 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1690029243735_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:34:04,734 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_1690029243735_ns3_test-16900290656252 to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029243735_ns3_test-16900290656252 2023-07-22T12:34:04,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=137 2023-07-22T12:34:05,549 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1690029243735_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:34:05,549 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1690029243735_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 }, snapshot procedure id = 137 2023-07-22T12:34:05,550 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=137, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1690029243735_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } in 1.8030 sec 2023-07-22T12:34:05,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=137 2023-07-22T12:34:05,853 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1690029237280 2023-07-22T12:34:05,853 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-07-22T12:34:05,854 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2023-07-22T12:34:05,854 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1690029237280_ns2:test-16900290656251 2023-07-22T12:34:05,854 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1690029241615_ns2_test-16900290656251 to hdfs://localhost:34745/backupUT/backup_1690029237280/ns2/test-16900290656251/ 2023-07-22T12:34:05,854 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-07-22T12:34:05,877 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:34745, inputRoot=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947 2023-07-22T12:34:05,877 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,37923,1690029053050_-1733762445_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:34745/backupUT/backup_1690029237280/ns2/test-16900290656251, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:34745/backupUT/backup_1690029237280/ns2/test-16900290656251/.hbase-snapshot/.tmp/snapshot_1690029241615_ns2_test-16900290656251 2023-07-22T12:34:05,877 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:34745, inputRoot=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947, snapshotDir=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029241615_ns2_test-16900290656251. 2023-07-22T12:34:05,885 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029241615_ns2_test-16900290656251 to hdfs://localhost:34745/backupUT/backup_1690029237280/ns2/test-16900290656251/.hbase-snapshot/.tmp/snapshot_1690029241615_ns2_test-16900290656251 2023-07-22T12:34:06,313 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:06,314 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:06,314 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:06,314 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:07,274 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/hadoop-587464005235840316.jar 2023-07-22T12:34:07,275 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:07,275 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:07,275 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:07,276 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:07,276 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:07,276 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:07,277 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:07,277 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:07,277 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:07,278 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:34:07,279 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-07-22T12:34:07,279 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-07-22T12:34:07,279 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-07-22T12:34:07,280 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-07-22T12:34:07,280 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-07-22T12:34:07,280 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-07-22T12:34:07,281 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-07-22T12:34:07,281 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-07-22T12:34:07,281 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-07-22T12:34:07,282 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-07-22T12:34:07,282 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-07-22T12:34:07,283 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:34:07,283 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:34:07,284 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:34:07,284 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:34:07,284 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:34:07,284 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:34:07,285 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:34:07,326 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1690029241615_ns2_test-16900290656251' hfile list 2023-07-22T12:34:07,328 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-07-22T12:34:07,407 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029247370/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-07-22T12:34:07,407 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029247370/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-07-22T12:34:07,463 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-07-22T12:34:07,487 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/01e992ceb654b1e583b990da186e85a8/f/d63801d92d8645ef95d1060b6ba3bbd9_SeqId_4_ output=hdfs://localhost:34745/backupUT/backup_1690029237280/ns2/test-16900290656251/archive/data/ns2/test-16900290656251/01e992ceb654b1e583b990da186e85a8/f/d63801d92d8645ef95d1060b6ba3bbd9_SeqId_4_ 2023-07-22T12:34:07,487 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8724 (8.5 K) time=0sec 8.320M/sec 2023-07-22T12:34:08,443 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-07-22T12:34:08,444 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-07-22T12:34:08,449 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1690029241615_ns2_test-16900290656251 2023-07-22T12:34:08,449 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1690029241615_ns2_test-16900290656251 finished. 2023-07-22T12:34:08,449 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1690029237280_ns3:test-16900290656252 2023-07-22T12:34:08,449 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1690029243735_ns3_test-16900290656252 to hdfs://localhost:34745/backupUT/backup_1690029237280/ns3/test-16900290656252/ 2023-07-22T12:34:08,450 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-07-22T12:34:08,472 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:34745, inputRoot=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947 2023-07-22T12:34:08,472 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,37923,1690029053050_1194183899_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:34745/backupUT/backup_1690029237280/ns3/test-16900290656252, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:34745/backupUT/backup_1690029237280/ns3/test-16900290656252/.hbase-snapshot/.tmp/snapshot_1690029243735_ns3_test-16900290656252 2023-07-22T12:34:08,472 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:34745, inputRoot=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947, snapshotDir=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029243735_ns3_test-16900290656252. 2023-07-22T12:34:08,477 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029243735_ns3_test-16900290656252 to hdfs://localhost:34745/backupUT/backup_1690029237280/ns3/test-16900290656252/.hbase-snapshot/.tmp/snapshot_1690029243735_ns3_test-16900290656252 2023-07-22T12:34:08,895 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:08,895 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:08,896 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:08,896 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:09,854 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/hadoop-7522005269655336820.jar 2023-07-22T12:34:09,854 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:09,854 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:09,855 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:09,855 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:09,855 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:09,856 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:09,856 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:09,856 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:09,857 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:09,857 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:34:09,858 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-07-22T12:34:09,858 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-07-22T12:34:09,859 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-07-22T12:34:09,859 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-07-22T12:34:09,859 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-07-22T12:34:09,860 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-07-22T12:34:09,860 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-07-22T12:34:09,860 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-07-22T12:34:09,861 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-07-22T12:34:09,861 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-07-22T12:34:09,861 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-07-22T12:34:09,862 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:34:09,862 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:34:09,863 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:34:09,863 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:34:09,864 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:34:09,864 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:34:09,865 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:34:09,905 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1690029243735_ns3_test-16900290656252' hfile list 2023-07-22T12:34:09,972 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029249940/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-07-22T12:34:09,972 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029249940/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-07-22T12:34:10,999 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-07-22T12:34:11,000 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-07-22T12:34:11,004 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1690029243735_ns3_test-16900290656252 2023-07-22T12:34:11,004 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1690029243735_ns3_test-16900290656252 finished. 2023-07-22T12:34:11,004 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(1014): ns2:test-16900290656251 2023-07-22T12:34:11,004 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(1014): ns3:test-16900290656252 2023-07-22T12:34:11,009 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029237280 2023-07-22T12:34:11,010 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-07-22T12:34:11,414 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:34745/backupUT/backup_1690029237280/.backup.manifest 2023-07-22T12:34:11,414 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029237280 2023-07-22T12:34:11,414 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-07-22T12:34:11,820 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:34745/backupUT/backup_1690029237280/.backup.manifest 2023-07-22T12:34:11,820 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(358): Backup backup_1690029237280 finished: type=FULL,tablelist=ns2:test-16900290656251;ns3:test-16900290656252,targetRootDir=hdfs://localhost:34745/backupUT,startts=1690029241422,completets=1690029251009,bytescopied=0 2023-07-22T12:34:11,820 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-07-22T12:34:11,820 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1690029241615_ns2_test-16900290656251 2023-07-22T12:34:11,821 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1690029241615_ns2_test-16900290656251" 2023-07-22T12:34:11,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1690029241615_ns2_test-16900290656251 2023-07-22T12:34:11,828 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1690029241615_ns2_test-16900290656251 for backup backup_1690029237280 succeeded. 2023-07-22T12:34:11,828 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1690029243735_ns3_test-16900290656252 2023-07-22T12:34:11,830 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1690029243735_ns3_test-16900290656252" 2023-07-22T12:34:11,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1690029243735_ns3_test-16900290656252 2023-07-22T12:34:11,833 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1690029243735_ns3_test-16900290656252 for backup backup_1690029237280 succeeded. 2023-07-22T12:34:11,834 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-07-22T12:34:11,838 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-07-22T12:34:11,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-07-22T12:34:11,841 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-07-22T12:34:11,842 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-07-22T12:34:11,848 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(376): Backup backup_1690029237280 completed. 2023-07-22T12:34:11,848 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:34:11,870 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fd2b53d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:34:11,871 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:34:11,873 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:40908, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:34:11,874 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(101): HBase table ns2:table2 does not exist. It will be created during restore process 2023-07-22T12:34:11,875 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(101): HBase table ns3:table3_restore does not exist. It will be created during restore process 2023-07-22T12:34:11,875 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34745/backupUT/backup_1690029237280/.backup.manifest 2023-07-22T12:34:11,877 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029237280/.backup.manifest 2023-07-22T12:34:11,878 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34745/backupUT/backup_1690029237280/.backup.manifest 2023-07-22T12:34:11,879 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029237280/.backup.manifest 2023-07-22T12:34:11,880 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34745/backupUT/backup_1690029237280/.backup.manifest 2023-07-22T12:34:11,881 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029237280/.backup.manifest 2023-07-22T12:34:11,881 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(148): Restoring 'ns2:test-16900290656251' to 'ns2:table2' from full backup image hdfs://localhost:34745/backupUT/backup_1690029237280/ns2/test-16900290656251 2023-07-22T12:34:11,888 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] util.RestoreTool(487): Creating target table 'ns2:table2' 2023-07-22T12:34:11,888 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] util.RestoreTool(391): Parsing region dir: hdfs://localhost:34745/backupUT/backup_1690029237280/ns2/test-16900290656251/archive/data/ns2/test-16900290656251/01e992ceb654b1e583b990da186e85a8 2023-07-22T12:34:11,889 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:34745/backupUT/backup_1690029237280/ns2/test-16900290656251/archive/data/ns2/test-16900290656251/01e992ceb654b1e583b990da186e85a8/f in region [hdfs://localhost:34745/backupUT/backup_1690029237280/ns2/test-16900290656251/archive/data/ns2/test-16900290656251/01e992ceb654b1e583b990da186e85a8] 2023-07-22T12:34:11,891 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] util.RestoreTool(442): Trying to figure out region boundaries hfile=hdfs://localhost:34745/backupUT/backup_1690029237280/ns2/test-16900290656251/archive/data/ns2/test-16900290656251/01e992ceb654b1e583b990da186e85a8/f/d63801d92d8645ef95d1060b6ba3bbd9_SeqId_4_ first=row0 last=row98 2023-07-22T12:34:11,891 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: d63801d92d8645ef95d1060b6ba3bbd9_SeqId_4_ 2023-07-22T12:34:11,892 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:34:11,894 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:57242, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:34:11,895 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'ns2:table2', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:34:11,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=139, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns2:table2 2023-07-22T12:34:11,899 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=139, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:34:11,899 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns2" qualifier: "table2" procId is: 139 2023-07-22T12:34:11,899 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=139, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:34:11,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=139 2023-07-22T12:34:12,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=139 2023-07-22T12:34:12,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=139 2023-07-22T12:34:12,306 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 8f5e8f244e81ee6a66b63d8eb95db45e, NAME => 'ns2:table2,,1690029251895.8f5e8f244e81ee6a66b63d8eb95db45e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:table2', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947 2023-07-22T12:34:12,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=139 2023-07-22T12:34:12,712 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:table2,,1690029251895.8f5e8f244e81ee6a66b63d8eb95db45e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:34:12,712 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1668): Closing 8f5e8f244e81ee6a66b63d8eb95db45e, disabling compactions & flushes 2023-07-22T12:34:12,712 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:table2,,1690029251895.8f5e8f244e81ee6a66b63d8eb95db45e. 2023-07-22T12:34:12,712 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:table2,,1690029251895.8f5e8f244e81ee6a66b63d8eb95db45e. 2023-07-22T12:34:12,712 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:table2,,1690029251895.8f5e8f244e81ee6a66b63d8eb95db45e. after waiting 0 ms 2023-07-22T12:34:12,712 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:table2,,1690029251895.8f5e8f244e81ee6a66b63d8eb95db45e. 2023-07-22T12:34:12,712 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1909): Closed ns2:table2,,1690029251895.8f5e8f244e81ee6a66b63d8eb95db45e. 2023-07-22T12:34:12,712 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1622): Region close journal for 8f5e8f244e81ee6a66b63d8eb95db45e: 2023-07-22T12:34:12,713 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=139, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:34:12,713 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:table2,,1690029251895.8f5e8f244e81ee6a66b63d8eb95db45e.","families":{"info":[{"qualifier":"regioninfo","vlen":36,"tag":[],"timestamp":"1690029252713"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690029252713"}]},"ts":"1690029252713"} 2023-07-22T12:34:12,714 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:34:12,715 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=139, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:34:12,715 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029252715"}]},"ts":"1690029252715"} 2023-07-22T12:34:12,716 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:table2, state=ENABLING in hbase:meta 2023-07-22T12:34:12,724 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:table2, region=8f5e8f244e81ee6a66b63d8eb95db45e, ASSIGN}] 2023-07-22T12:34:12,726 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=140, ppid=139, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:table2, region=8f5e8f244e81ee6a66b63d8eb95db45e, ASSIGN 2023-07-22T12:34:12,726 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=140, ppid=139, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:table2, region=8f5e8f244e81ee6a66b63d8eb95db45e, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,43639,1690029056327; forceNewPlan=false, retain=false 2023-07-22T12:34:12,878 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=140 updating hbase:meta row=8f5e8f244e81ee6a66b63d8eb95db45e, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:12,879 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE; OpenRegionProcedure 8f5e8f244e81ee6a66b63d8eb95db45e, server=jenkins-hbase3.apache.org,43639,1690029056327}] 2023-07-22T12:34:13,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=139 2023-07-22T12:34:13,034 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] handler.AssignRegionHandler(130): Open ns2:table2,,1690029251895.8f5e8f244e81ee6a66b63d8eb95db45e. 2023-07-22T12:34:13,034 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7254): Opening region: {ENCODED => 8f5e8f244e81ee6a66b63d8eb95db45e, NAME => 'ns2:table2,,1690029251895.8f5e8f244e81ee6a66b63d8eb95db45e.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:34:13,035 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:34:13,035 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table2 8f5e8f244e81ee6a66b63d8eb95db45e 2023-07-22T12:34:13,035 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(888): Instantiated ns2:table2,,1690029251895.8f5e8f244e81ee6a66b63d8eb95db45e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:34:13,035 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7296): checking encryption for 8f5e8f244e81ee6a66b63d8eb95db45e 2023-07-22T12:34:13,035 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7299): checking classloading for 8f5e8f244e81ee6a66b63d8eb95db45e 2023-07-22T12:34:13,036 INFO [StoreOpener-8f5e8f244e81ee6a66b63d8eb95db45e-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 8f5e8f244e81ee6a66b63d8eb95db45e 2023-07-22T12:34:13,038 INFO [StoreOpener-8f5e8f244e81ee6a66b63d8eb95db45e-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f5e8f244e81ee6a66b63d8eb95db45e columnFamilyName f 2023-07-22T12:34:13,038 DEBUG [StoreOpener-8f5e8f244e81ee6a66b63d8eb95db45e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:34:13,038 INFO [StoreOpener-8f5e8f244e81ee6a66b63d8eb95db45e-1 {}] regionserver.HStore(324): Store=8f5e8f244e81ee6a66b63d8eb95db45e/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:34:13,039 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/table2/8f5e8f244e81ee6a66b63d8eb95db45e 2023-07-22T12:34:13,039 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/table2/8f5e8f244e81ee6a66b63d8eb95db45e 2023-07-22T12:34:13,041 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1076): writing seq id for 8f5e8f244e81ee6a66b63d8eb95db45e 2023-07-22T12:34:13,043 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/table2/8f5e8f244e81ee6a66b63d8eb95db45e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:34:13,044 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1093): Opened 8f5e8f244e81ee6a66b63d8eb95db45e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10665932320, jitterRate=-0.006657645106315613}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-07-22T12:34:13,044 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(992): Region open journal for 8f5e8f244e81ee6a66b63d8eb95db45e: 2023-07-22T12:34:13,045 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegionServer(2552): Post open deploy tasks for ns2:table2,,1690029251895.8f5e8f244e81ee6a66b63d8eb95db45e., pid=141, masterSystemTime=1690029253031 2023-07-22T12:34:13,046 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegionServer(2579): Finished post open deploy task for ns2:table2,,1690029251895.8f5e8f244e81ee6a66b63d8eb95db45e. 2023-07-22T12:34:13,046 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] handler.AssignRegionHandler(158): Opened ns2:table2,,1690029251895.8f5e8f244e81ee6a66b63d8eb95db45e. 2023-07-22T12:34:13,047 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=140 updating hbase:meta row=8f5e8f244e81ee6a66b63d8eb95db45e, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:13,049 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=141, resume processing ppid=140 2023-07-22T12:34:13,049 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=141, ppid=140, state=SUCCESS; OpenRegionProcedure 8f5e8f244e81ee6a66b63d8eb95db45e, server=jenkins-hbase3.apache.org,43639,1690029056327 in 169 msec 2023-07-22T12:34:13,050 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=140, resume processing ppid=139 2023-07-22T12:34:13,050 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=140, ppid=139, state=SUCCESS; TransitRegionStateProcedure table=ns2:table2, region=8f5e8f244e81ee6a66b63d8eb95db45e, ASSIGN in 325 msec 2023-07-22T12:34:13,051 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=139, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:34:13,051 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029253051"}]},"ts":"1690029253051"} 2023-07-22T12:34:13,052 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:table2, state=ENABLED in hbase:meta 2023-07-22T12:34:13,054 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=139, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:34:13,055 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=139, state=SUCCESS; CreateTableProcedure table=ns2:table2 in 1.1590 sec 2023-07-22T12:34:13,225 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_table2 2023-07-22T12:34:13,225 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_table2 Metrics about Tables on a single HBase RegionServer 2023-07-22T12:34:14,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=139 2023-07-22T12:34:14,009 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:34745/backupUT/backup_1690029237280/ns2/test-16900290656251/archive/data/ns2/test-16900290656251/01e992ceb654b1e583b990da186e85a8 from hbase tables ns2:test-16900290656251 to tables ns2:table2 2023-07-22T12:34:14,009 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceRestoreJob(72): Restore ns2:test-16900290656251 into ns2:table2 2023-07-22T12:34:14,011 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-ns2-table2-1690029254009 from hdfs://localhost:34745/backupUT/backup_1690029237280/ns2/test-16900290656251/archive/data/ns2/test-16900290656251/01e992ceb654b1e583b990da186e85a8 2023-07-22T12:34:14,015 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4faef678, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:34:14,016 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:34:14,017 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:57254, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:34:14,018 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-07-22T12:34:14,018 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:table2 2023-07-22T12:34:14,021 DEBUG [hconnection-0x1376a5e8-shared-pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:34:14,021 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:40916, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:34:14,023 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:table2: 2023-07-22T12:34:14,023 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-07-22T12:34:14,023 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/partitions_f22b2ad4-b707-4aea-98a5-6a7da0757e49 2023-07-22T12:34:14,431 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:14,431 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:14,431 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:14,432 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:15,427 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/hadoop-260906042078297640.jar 2023-07-22T12:34:15,427 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:15,427 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:15,428 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:15,428 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:15,428 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:15,429 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:15,429 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:15,429 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:15,430 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:15,430 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:34:15,431 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-07-22T12:34:15,431 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-07-22T12:34:15,432 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-07-22T12:34:15,432 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-07-22T12:34:15,432 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-07-22T12:34:15,433 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-07-22T12:34:15,433 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-07-22T12:34:15,433 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-07-22T12:34:15,434 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-07-22T12:34:15,434 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-07-22T12:34:15,434 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-07-22T12:34:15,435 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:15,435 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:15,436 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:15,436 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:15,436 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:15,436 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:15,437 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:34:15,437 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: ns2:table2 2023-07-22T12:34:15,437 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:34:15,437 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-07-22T12:34:15,438 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-07-22T12:34:15,438 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:34:15,449 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-07-22T12:34:15,545 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029255508/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-07-22T12:34:15,545 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029255508/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-07-22T12:34:15,613 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:34745/backupUT/backup_1690029237280/ns2/test-16900290656251/archive/data/ns2/test-16900290656251/01e992ceb654b1e583b990da186e85a8/f/d63801d92d8645ef95d1060b6ba3bbd9_SeqId_4_ 2023-07-22T12:34:15,617 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-07-22T12:34:15,618 DEBUG [LocalJobRunner Map Task Executor #0 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: d63801d92d8645ef95d1060b6ba3bbd9_SeqId_4_ 2023-07-22T12:34:15,635 DEBUG [pool-482-thread-1 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c6fd392, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:34:15,636 DEBUG [hconnection-0x3cdc7f55-metaLookup-shared--pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:34:15,638 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:40930, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:34:15,641 DEBUG [pool-482-thread-1 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:34:15,641 DEBUG [pool-482-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-07-22T12:34:15,641 DEBUG [pool-482-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-07-22T12:34:15,645 DEBUG [pool-482-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-ns2-table2-1690029254009/_temporary/0/_temporary/attempt_local2065660696_0022_r_000000_0/f/c24133b407944573ac1f3d153b6fb606 is 37, key is row10/f:q1/1690029070502/Put/seqid=0 2023-07-22T12:34:15,647 WARN [Thread-2313 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:43639] Specified favored nodes: [jenkins-hbase3.apache.org:43639] 2023-07-22T12:34:16,597 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-ns2-table2-1690029254009 2023-07-22T12:34:16,602 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@efef481, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:34:16,603 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:34:16,605 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:40932, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:34:16,610 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-ns2-table2-1690029254009/_SUCCESS 2023-07-22T12:34:16,613 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:34:16,614 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:57256, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:34:16,622 DEBUG [LoadIncrementalHFiles-0 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: c24133b407944573ac1f3d153b6fb606 2023-07-22T12:34:16,622 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-ns2-table2-1690029254009/f/c24133b407944573ac1f3d153b6fb606 first=Optional[row0] last=Optional[row98] 2023-07-22T12:34:16,623 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=ns2:table2,,1690029251895.8f5e8f244e81ee6a66b63d8eb95db45e., hostname=jenkins-hbase3.apache.org,43639,1690029056327, seqNum=2 for row with hfile group [{f,hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-ns2-table2-1690029254009/f/c24133b407944573ac1f3d153b6fb606}] 2023-07-22T12:34:16,628 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-ns2-table2-1690029254009/f/c24133b407944573ac1f3d153b6fb606 for inclusion in 8f5e8f244e81ee6a66b63d8eb95db45e/f 2023-07-22T12:34:16,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-07-22T12:34:16,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HStore(625): Region bounds: first= last= 2023-07-22T12:34:16,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: c24133b407944573ac1f3d153b6fb606 2023-07-22T12:34:16,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HRegion(2520): Flush status journal for 8f5e8f244e81ee6a66b63d8eb95db45e: 2023-07-22T12:34:16,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-ns2-table2-1690029254009/f/c24133b407944573ac1f3d153b6fb606 to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/staging/jenkins__ns2_table2__ta8esim2e8v2de41qhgvpkdcrteqe9f4rkbrmchk3tf3mjklj8i6op6326op44td/f/c24133b407944573ac1f3d153b6fb606 2023-07-22T12:34:16,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/staging/jenkins__ns2_table2__ta8esim2e8v2de41qhgvpkdcrteqe9f4rkbrmchk3tf3mjklj8i6op6326op44td/f/c24133b407944573ac1f3d153b6fb606 as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/table2/8f5e8f244e81ee6a66b63d8eb95db45e/f/21b4baeba3664c039e9104ad262b07e1_SeqId_4_ 2023-07-22T12:34:16,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@366d7c07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:34:16,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:34:16,640 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:57270, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:34:16,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:34:16,643 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:40944, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:34:16,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:34:16,654 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/staging/jenkins__ns2_table2__ta8esim2e8v2de41qhgvpkdcrteqe9f4rkbrmchk3tf3mjklj8i6op6326op44td/f/c24133b407944573ac1f3d153b6fb606 into 8f5e8f244e81ee6a66b63d8eb95db45e/f as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/table2/8f5e8f244e81ee6a66b63d8eb95db45e/f/21b4baeba3664c039e9104ad262b07e1_SeqId_4_ - updating store file list. 2023-07-22T12:34:16,658 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/table2/8f5e8f244e81ee6a66b63d8eb95db45e/f/21b4baeba3664c039e9104ad262b07e1_SeqId_4_ into 8f5e8f244e81ee6a66b63d8eb95db45e/f 2023-07-22T12:34:16,658 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/staging/jenkins__ns2_table2__ta8esim2e8v2de41qhgvpkdcrteqe9f4rkbrmchk3tf3mjklj8i6op6326op44td/f/c24133b407944573ac1f3d153b6fb606 into 8f5e8f244e81ee6a66b63d8eb95db45e/f (new location: hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/table2/8f5e8f244e81ee6a66b63d8eb95db45e/f/21b4baeba3664c039e9104ad262b07e1_SeqId_4_) 2023-07-22T12:34:16,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/staging/jenkins__ns2_table2__ta8esim2e8v2de41qhgvpkdcrteqe9f4rkbrmchk3tf3mjklj8i6op6326op44td/f/c24133b407944573ac1f3d153b6fb606 2023-07-22T12:34:16,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64bd03e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:34:16,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:34:16,671 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:57272, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-07-22T12:34:16,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:34:16,675 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:40946, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-07-22T12:34:16,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:34:16,687 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:34:16,687 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-07-22T12:34:16,687 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-07-22T12:34:16,687 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(233): Backup: backup_1690029237280 hdfs://localhost:34745/backupUT/backup_1690029237280/ns2/test-16900290656251/ 2023-07-22T12:34:16,688 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34745/backupUT/backup_1690029237280/.backup.manifest 2023-07-22T12:34:16,690 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029237280/.backup.manifest 2023-07-22T12:34:16,690 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(148): Restoring 'ns3:test-16900290656252' to 'ns3:table3_restore' from full backup image hdfs://localhost:34745/backupUT/backup_1690029237280/ns3/test-16900290656252 2023-07-22T12:34:16,694 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] util.RestoreTool(101): Folder tableArchivePath: hdfs://localhost:34745/backupUT/backup_1690029237280/ns3/test-16900290656252/archive/data/ns3/test-16900290656252 does not exists 2023-07-22T12:34:16,694 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] util.RestoreTool(326): find table descriptor but no archive dir for table ns3:test-16900290656252, will only create table 2023-07-22T12:34:16,695 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] util.RestoreTool(487): Creating target table 'ns3:table3_restore' 2023-07-22T12:34:16,696 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'ns3:table3_restore', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:34:16,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns3:table3_restore 2023-07-22T12:34:16,698 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:34:16,698 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns3" qualifier: "table3_restore" procId is: 142 2023-07-22T12:34:16,699 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=142, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:34:16,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=142 2023-07-22T12:34:16,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=142 2023-07-22T12:34:17,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=142 2023-07-22T12:34:17,105 INFO [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => e6e2c196af75d812d914d1af7ae119bd, NAME => 'ns3:table3_restore,,1690029256696.e6e2c196af75d812d914d1af7ae119bd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:table3_restore', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947 2023-07-22T12:34:17,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=142 2023-07-22T12:34:17,510 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:table3_restore,,1690029256696.e6e2c196af75d812d914d1af7ae119bd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:34:17,511 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1668): Closing e6e2c196af75d812d914d1af7ae119bd, disabling compactions & flushes 2023-07-22T12:34:17,511 INFO [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:table3_restore,,1690029256696.e6e2c196af75d812d914d1af7ae119bd. 2023-07-22T12:34:17,511 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:table3_restore,,1690029256696.e6e2c196af75d812d914d1af7ae119bd. 2023-07-22T12:34:17,511 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:table3_restore,,1690029256696.e6e2c196af75d812d914d1af7ae119bd. after waiting 0 ms 2023-07-22T12:34:17,511 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:table3_restore,,1690029256696.e6e2c196af75d812d914d1af7ae119bd. 2023-07-22T12:34:17,511 INFO [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1909): Closed ns3:table3_restore,,1690029256696.e6e2c196af75d812d914d1af7ae119bd. 2023-07-22T12:34:17,511 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1622): Region close journal for e6e2c196af75d812d914d1af7ae119bd: 2023-07-22T12:34:17,512 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=142, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:34:17,512 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:table3_restore,,1690029256696.e6e2c196af75d812d914d1af7ae119bd.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1690029257512"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690029257512"}]},"ts":"1690029257512"} 2023-07-22T12:34:17,513 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:34:17,514 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=142, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:34:17,514 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029257514"}]},"ts":"1690029257514"} 2023-07-22T12:34:17,515 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:table3_restore, state=ENABLING in hbase:meta 2023-07-22T12:34:17,516 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:table2' 2023-07-22T12:34:17,521 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:table3_restore, region=e6e2c196af75d812d914d1af7ae119bd, ASSIGN}] 2023-07-22T12:34:17,522 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:table3_restore, region=e6e2c196af75d812d914d1af7ae119bd, ASSIGN 2023-07-22T12:34:17,523 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:table3_restore, region=e6e2c196af75d812d914d1af7ae119bd, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,43639,1690029056327; forceNewPlan=false, retain=false 2023-07-22T12:34:17,675 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=143 updating hbase:meta row=e6e2c196af75d812d914d1af7ae119bd, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:17,677 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE; OpenRegionProcedure e6e2c196af75d812d914d1af7ae119bd, server=jenkins-hbase3.apache.org,43639,1690029056327}] 2023-07-22T12:34:17,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=142 2023-07-22T12:34:17,831 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(130): Open ns3:table3_restore,,1690029256696.e6e2c196af75d812d914d1af7ae119bd. 2023-07-22T12:34:17,831 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7254): Opening region: {ENCODED => e6e2c196af75d812d914d1af7ae119bd, NAME => 'ns3:table3_restore,,1690029256696.e6e2c196af75d812d914d1af7ae119bd.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:34:17,831 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:34:17,831 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table3_restore e6e2c196af75d812d914d1af7ae119bd 2023-07-22T12:34:17,832 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(888): Instantiated ns3:table3_restore,,1690029256696.e6e2c196af75d812d914d1af7ae119bd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:34:17,832 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7296): checking encryption for e6e2c196af75d812d914d1af7ae119bd 2023-07-22T12:34:17,832 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7299): checking classloading for e6e2c196af75d812d914d1af7ae119bd 2023-07-22T12:34:17,833 INFO [StoreOpener-e6e2c196af75d812d914d1af7ae119bd-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region e6e2c196af75d812d914d1af7ae119bd 2023-07-22T12:34:17,834 INFO [StoreOpener-e6e2c196af75d812d914d1af7ae119bd-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e6e2c196af75d812d914d1af7ae119bd columnFamilyName f 2023-07-22T12:34:17,834 DEBUG [StoreOpener-e6e2c196af75d812d914d1af7ae119bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:34:17,834 INFO [StoreOpener-e6e2c196af75d812d914d1af7ae119bd-1 {}] regionserver.HStore(324): Store=e6e2c196af75d812d914d1af7ae119bd/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:34:17,835 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns3/table3_restore/e6e2c196af75d812d914d1af7ae119bd 2023-07-22T12:34:17,835 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns3/table3_restore/e6e2c196af75d812d914d1af7ae119bd 2023-07-22T12:34:17,837 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1076): writing seq id for e6e2c196af75d812d914d1af7ae119bd 2023-07-22T12:34:17,839 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns3/table3_restore/e6e2c196af75d812d914d1af7ae119bd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:34:17,839 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1093): Opened e6e2c196af75d812d914d1af7ae119bd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10414188640, jitterRate=-0.030103102326393127}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-07-22T12:34:17,840 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(992): Region open journal for e6e2c196af75d812d914d1af7ae119bd: 2023-07-22T12:34:17,841 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2552): Post open deploy tasks for ns3:table3_restore,,1690029256696.e6e2c196af75d812d914d1af7ae119bd., pid=144, masterSystemTime=1690029257828 2023-07-22T12:34:17,842 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2579): Finished post open deploy task for ns3:table3_restore,,1690029256696.e6e2c196af75d812d914d1af7ae119bd. 2023-07-22T12:34:17,842 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(158): Opened ns3:table3_restore,,1690029256696.e6e2c196af75d812d914d1af7ae119bd. 2023-07-22T12:34:17,843 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=143 updating hbase:meta row=e6e2c196af75d812d914d1af7ae119bd, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:17,845 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=144, resume processing ppid=143 2023-07-22T12:34:17,846 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=144, ppid=143, state=SUCCESS; OpenRegionProcedure e6e2c196af75d812d914d1af7ae119bd, server=jenkins-hbase3.apache.org,43639,1690029056327 in 168 msec 2023-07-22T12:34:17,847 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=143, resume processing ppid=142 2023-07-22T12:34:17,847 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=143, ppid=142, state=SUCCESS; TransitRegionStateProcedure table=ns3:table3_restore, region=e6e2c196af75d812d914d1af7ae119bd, ASSIGN in 324 msec 2023-07-22T12:34:17,848 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=142, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:34:17,848 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029257848"}]},"ts":"1690029257848"} 2023-07-22T12:34:17,849 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:table3_restore, state=ENABLED in hbase:meta 2023-07-22T12:34:17,851 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=142, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:34:17,852 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=142, state=SUCCESS; CreateTableProcedure table=ns3:table3_restore in 1.1550 sec 2023-07-22T12:34:18,576 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-07-22T12:34:18,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=142 2023-07-22T12:34:18,806 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-07-22T12:34:18,806 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(233): Backup: backup_1690029237280 hdfs://localhost:34745/backupUT/backup_1690029237280/ns3/test-16900290656252/ 2023-07-22T12:34:18,806 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-07-22T12:34:18,806 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:34:18,808 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster$12(2663): Client=jenkins//172.31.12.81 disable ns2:table2 2023-07-22T12:34:18,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=145, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=ns2:table2 2023-07-22T12:34:18,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=145 2023-07-22T12:34:18,812 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029258812"}]},"ts":"1690029258812"} 2023-07-22T12:34:18,813 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:table2, state=DISABLING in hbase:meta 2023-07-22T12:34:18,814 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(293): Set ns2:table2 to state=DISABLING 2023-07-22T12:34:18,814 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns2:table2, region=8f5e8f244e81ee6a66b63d8eb95db45e, UNASSIGN}] 2023-07-22T12:34:18,816 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=146, ppid=145, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns2:table2, region=8f5e8f244e81ee6a66b63d8eb95db45e, UNASSIGN 2023-07-22T12:34:18,817 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=146 updating hbase:meta row=8f5e8f244e81ee6a66b63d8eb95db45e, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:18,818 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE; CloseRegionProcedure 8f5e8f244e81ee6a66b63d8eb95db45e, server=jenkins-hbase3.apache.org,43639,1690029056327}] 2023-07-22T12:34:18,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=145 2023-07-22T12:34:18,969 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] handler.UnassignRegionHandler(119): Close 8f5e8f244e81ee6a66b63d8eb95db45e 2023-07-22T12:34:18,970 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1668): Closing 8f5e8f244e81ee6a66b63d8eb95db45e, disabling compactions & flushes 2023-07-22T12:34:18,970 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1690): Closing region ns2:table2,,1690029251895.8f5e8f244e81ee6a66b63d8eb95db45e. 2023-07-22T12:34:18,970 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:table2,,1690029251895.8f5e8f244e81ee6a66b63d8eb95db45e. 2023-07-22T12:34:18,970 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1778): Acquired close lock on ns2:table2,,1690029251895.8f5e8f244e81ee6a66b63d8eb95db45e. after waiting 0 ms 2023-07-22T12:34:18,970 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1788): Updates disabled for region ns2:table2,,1690029251895.8f5e8f244e81ee6a66b63d8eb95db45e. 2023-07-22T12:34:18,971 DEBUG [StoreFileCloser-8f5e8f244e81ee6a66b63d8eb95db45e-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 21b4baeba3664c039e9104ad262b07e1_SeqId_4_ 2023-07-22T12:34:18,975 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/table2/8f5e8f244e81ee6a66b63d8eb95db45e/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2023-07-22T12:34:18,976 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:34:18,976 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1909): Closed ns2:table2,,1690029251895.8f5e8f244e81ee6a66b63d8eb95db45e. 2023-07-22T12:34:18,976 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1622): Region close journal for 8f5e8f244e81ee6a66b63d8eb95db45e: 2023-07-22T12:34:18,978 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] handler.UnassignRegionHandler(163): Closed 8f5e8f244e81ee6a66b63d8eb95db45e 2023-07-22T12:34:18,978 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=146 updating hbase:meta row=8f5e8f244e81ee6a66b63d8eb95db45e, regionState=CLOSED 2023-07-22T12:34:18,980 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=147, resume processing ppid=146 2023-07-22T12:34:18,980 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=147, ppid=146, state=SUCCESS; CloseRegionProcedure 8f5e8f244e81ee6a66b63d8eb95db45e, server=jenkins-hbase3.apache.org,43639,1690029056327 in 161 msec 2023-07-22T12:34:18,981 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=146, resume processing ppid=145 2023-07-22T12:34:18,981 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=146, ppid=145, state=SUCCESS; TransitRegionStateProcedure table=ns2:table2, region=8f5e8f244e81ee6a66b63d8eb95db45e, UNASSIGN in 166 msec 2023-07-22T12:34:18,981 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029258981"}]},"ts":"1690029258981"} 2023-07-22T12:34:18,982 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:table2, state=DISABLED in hbase:meta 2023-07-22T12:34:18,985 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(305): Set ns2:table2 to state=DISABLED 2023-07-22T12:34:18,986 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=145, state=SUCCESS; DisableTableProcedure table=ns2:table2 in 177 msec 2023-07-22T12:34:19,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=145 2023-07-22T12:34:19,114 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster$5(2447): Client=jenkins//172.31.12.81 delete ns2:table2 2023-07-22T12:34:19,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=148, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=ns2:table2 2023-07-22T12:34:19,116 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=148, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=ns2:table2 2023-07-22T12:34:19,117 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=148, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=ns2:table2 2023-07-22T12:34:19,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=148 2023-07-22T12:34:19,118 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/table2/8f5e8f244e81ee6a66b63d8eb95db45e 2023-07-22T12:34:19,120 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/table2/8f5e8f244e81ee6a66b63d8eb95db45e/f, FileablePath, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/table2/8f5e8f244e81ee6a66b63d8eb95db45e/recovered.edits] 2023-07-22T12:34:19,123 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/table2/8f5e8f244e81ee6a66b63d8eb95db45e/f/21b4baeba3664c039e9104ad262b07e1_SeqId_4_ to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/archive/data/ns2/table2/8f5e8f244e81ee6a66b63d8eb95db45e/f/21b4baeba3664c039e9104ad262b07e1_SeqId_4_ 2023-07-22T12:34:19,125 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/table2/8f5e8f244e81ee6a66b63d8eb95db45e/recovered.edits/6.seqid to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/archive/data/ns2/table2/8f5e8f244e81ee6a66b63d8eb95db45e/recovered.edits/6.seqid 2023-07-22T12:34:19,126 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/table2/8f5e8f244e81ee6a66b63d8eb95db45e 2023-07-22T12:34:19,126 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(293): Archived ns2:table2 regions 2023-07-22T12:34:19,127 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=148, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=ns2:table2 2023-07-22T12:34:19,129 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of ns2:table2 from hbase:meta 2023-07-22T12:34:19,130 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(387): Removing 'ns2:table2' descriptor. 2023-07-22T12:34:19,131 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=148, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=ns2:table2 2023-07-22T12:34:19,131 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(377): Removing 'ns2:table2' from region states. 2023-07-22T12:34:19,131 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns2:table2,,1690029251895.8f5e8f244e81ee6a66b63d8eb95db45e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1690029259131"}]},"ts":"9223372036854775807"} 2023-07-22T12:34:19,133 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-07-22T12:34:19,133 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 8f5e8f244e81ee6a66b63d8eb95db45e, NAME => 'ns2:table2,,1690029251895.8f5e8f244e81ee6a66b63d8eb95db45e.', STARTKEY => '', ENDKEY => ''}] 2023-07-22T12:34:19,133 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(381): Marking 'ns2:table2' as deleted. 2023-07-22T12:34:19,133 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1690029259133"}]},"ts":"9223372036854775807"} 2023-07-22T12:34:19,134 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1678): Deleted table ns2:table2 state from META 2023-07-22T12:34:19,136 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(127): Finished pid=148, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=ns2:table2 2023-07-22T12:34:19,136 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=148, state=SUCCESS; DeleteTableProcedure table=ns2:table2 in 22 msec 2023-07-22T12:34:19,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=148 2023-07-22T12:34:19,218 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster$12(2663): Client=jenkins//172.31.12.81 disable ns3:table3_restore 2023-07-22T12:34:19,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=149, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=ns3:table3_restore 2023-07-22T12:34:19,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=149 2023-07-22T12:34:19,222 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029259222"}]},"ts":"1690029259222"} 2023-07-22T12:34:19,223 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:table3_restore, state=DISABLING in hbase:meta 2023-07-22T12:34:19,224 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(293): Set ns3:table3_restore to state=DISABLING 2023-07-22T12:34:19,225 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns3:table3_restore, region=e6e2c196af75d812d914d1af7ae119bd, UNASSIGN}] 2023-07-22T12:34:19,226 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=150, ppid=149, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns3:table3_restore, region=e6e2c196af75d812d914d1af7ae119bd, UNASSIGN 2023-07-22T12:34:19,226 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=150 updating hbase:meta row=e6e2c196af75d812d914d1af7ae119bd, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:19,227 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE; CloseRegionProcedure e6e2c196af75d812d914d1af7ae119bd, server=jenkins-hbase3.apache.org,43639,1690029056327}] 2023-07-22T12:34:19,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=149 2023-07-22T12:34:19,379 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] handler.UnassignRegionHandler(119): Close e6e2c196af75d812d914d1af7ae119bd 2023-07-22T12:34:19,380 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(1668): Closing e6e2c196af75d812d914d1af7ae119bd, disabling compactions & flushes 2023-07-22T12:34:19,380 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(1690): Closing region ns3:table3_restore,,1690029256696.e6e2c196af75d812d914d1af7ae119bd. 2023-07-22T12:34:19,380 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:table3_restore,,1690029256696.e6e2c196af75d812d914d1af7ae119bd. 2023-07-22T12:34:19,380 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(1778): Acquired close lock on ns3:table3_restore,,1690029256696.e6e2c196af75d812d914d1af7ae119bd. after waiting 0 ms 2023-07-22T12:34:19,380 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(1788): Updates disabled for region ns3:table3_restore,,1690029256696.e6e2c196af75d812d914d1af7ae119bd. 2023-07-22T12:34:19,383 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns3/table3_restore/e6e2c196af75d812d914d1af7ae119bd/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-07-22T12:34:19,384 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:34:19,385 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(1909): Closed ns3:table3_restore,,1690029256696.e6e2c196af75d812d914d1af7ae119bd. 2023-07-22T12:34:19,385 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(1622): Region close journal for e6e2c196af75d812d914d1af7ae119bd: 2023-07-22T12:34:19,386 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] handler.UnassignRegionHandler(163): Closed e6e2c196af75d812d914d1af7ae119bd 2023-07-22T12:34:19,386 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=150 updating hbase:meta row=e6e2c196af75d812d914d1af7ae119bd, regionState=CLOSED 2023-07-22T12:34:19,388 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=151, resume processing ppid=150 2023-07-22T12:34:19,388 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=151, ppid=150, state=SUCCESS; CloseRegionProcedure e6e2c196af75d812d914d1af7ae119bd, server=jenkins-hbase3.apache.org,43639,1690029056327 in 160 msec 2023-07-22T12:34:19,389 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=150, resume processing ppid=149 2023-07-22T12:34:19,389 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=150, ppid=149, state=SUCCESS; TransitRegionStateProcedure table=ns3:table3_restore, region=e6e2c196af75d812d914d1af7ae119bd, UNASSIGN in 164 msec 2023-07-22T12:34:19,390 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029259390"}]},"ts":"1690029259390"} 2023-07-22T12:34:19,391 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:table3_restore, state=DISABLED in hbase:meta 2023-07-22T12:34:19,392 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(305): Set ns3:table3_restore to state=DISABLED 2023-07-22T12:34:19,393 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=149, state=SUCCESS; DisableTableProcedure table=ns3:table3_restore in 175 msec 2023-07-22T12:34:19,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=149 2023-07-22T12:34:19,524 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster$5(2447): Client=jenkins//172.31.12.81 delete ns3:table3_restore 2023-07-22T12:34:19,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=152, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=ns3:table3_restore 2023-07-22T12:34:19,526 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=152, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-07-22T12:34:19,527 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=152, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-07-22T12:34:19,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=152 2023-07-22T12:34:19,529 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns3/table3_restore/e6e2c196af75d812d914d1af7ae119bd 2023-07-22T12:34:19,530 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns3/table3_restore/e6e2c196af75d812d914d1af7ae119bd/f, FileablePath, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns3/table3_restore/e6e2c196af75d812d914d1af7ae119bd/recovered.edits] 2023-07-22T12:34:19,535 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns3/table3_restore/e6e2c196af75d812d914d1af7ae119bd/recovered.edits/4.seqid to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/archive/data/ns3/table3_restore/e6e2c196af75d812d914d1af7ae119bd/recovered.edits/4.seqid 2023-07-22T12:34:19,535 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns3/table3_restore/e6e2c196af75d812d914d1af7ae119bd 2023-07-22T12:34:19,535 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(293): Archived ns3:table3_restore regions 2023-07-22T12:34:19,537 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=152, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-07-22T12:34:19,539 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of ns3:table3_restore from hbase:meta 2023-07-22T12:34:19,541 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(387): Removing 'ns3:table3_restore' descriptor. 2023-07-22T12:34:19,541 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=152, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-07-22T12:34:19,542 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(377): Removing 'ns3:table3_restore' from region states. 2023-07-22T12:34:19,542 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns3:table3_restore,,1690029256696.e6e2c196af75d812d914d1af7ae119bd.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1690029259542"}]},"ts":"9223372036854775807"} 2023-07-22T12:34:19,543 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-07-22T12:34:19,543 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => e6e2c196af75d812d914d1af7ae119bd, NAME => 'ns3:table3_restore,,1690029256696.e6e2c196af75d812d914d1af7ae119bd.', STARTKEY => '', ENDKEY => ''}] 2023-07-22T12:34:19,543 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(381): Marking 'ns3:table3_restore' as deleted. 2023-07-22T12:34:19,543 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1690029259543"}]},"ts":"9223372036854775807"} 2023-07-22T12:34:19,544 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1678): Deleted table ns3:table3_restore state from META 2023-07-22T12:34:19,547 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(127): Finished pid=152, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-07-22T12:34:19,548 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=152, state=SUCCESS; DeleteTableProcedure table=ns3:table3_restore in 23 msec 2023-07-22T12:34:19,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=152 2023-07-22T12:34:19,645 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreMultipleCommand Thread=604 (was 606), OpenFileDescriptor=679 (was 686), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=96 (was 113), ProcessCount=169 (was 169), AvailableMemoryMB=16232 (was 16393) 2023-07-22T12:34:19,645 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(130): Thread=604 is superior to 500 2023-07-22T12:34:19,659 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreSingleDNE Thread=604, OpenFileDescriptor=679, MaxFileDescriptor=60000, SystemLoadAverage=96, ProcessCount=169, AvailableMemoryMB=16232 2023-07-22T12:34:19,659 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(130): Thread=604 is superior to 500 2023-07-22T12:34:19,660 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] backup.TestFullRestore(260): test restore fails on a single table that does not exist 2023-07-22T12:34:19,664 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cd20cb4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:34:19,666 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:34:19,668 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:40962, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:34:19,669 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:34:19,670 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:57284, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:34:19,679 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-07-22T12:34:19,687 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:34:19,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029259687 (current time:1690029259687). 2023-07-22T12:34:19,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:34:19,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-07-22T12:34:19,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41d6c745, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:34:19,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:34:19,700 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:40976, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:34:19,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:34:19,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:34:19,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=153, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:34:19,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 153 2023-07-22T12:34:19,704 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:34:19,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=153 2023-07-22T12:34:19,705 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:34:19,706 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:34:19,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=153 2023-07-22T12:34:20,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=153 2023-07-22T12:34:20,113 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:34:20,113 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33}] 2023-07-22T12:34:20,115 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=154, ppid=153, state=RUNNABLE; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33 2023-07-22T12:34:20,266 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43639 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=154 2023-07-22T12:34:20,266 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. 2023-07-22T12:34:20,266 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.HRegion(2819): Flushing 697cc5ecf0962f5ab9f4126febf7ce33 2/2 column families, dataSize=1.50 KB heapSize=2.73 KB 2023-07-22T12:34:20,270 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/meta/93cd26fa4bd94fc8b3a8b6de32bded79 is 171, key is trslm:hdfs://localhost:34745/backupUT\x00ns2:test-16900290656251/meta:log-roll-map/1690029251006/Put/seqid=0 2023-07-22T12:34:20,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=153 2023-07-22T12:34:20,674 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=717 B at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/meta/93cd26fa4bd94fc8b3a8b6de32bded79 2023-07-22T12:34:20,678 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 93cd26fa4bd94fc8b3a8b6de32bded79 2023-07-22T12:34:20,681 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/session/20a6ddbb9c214d998c32b0bb0820f924 is 524, key is session:backup_1690029237280/session:context/1690029251842/Put/seqid=0 2023-07-22T12:34:20,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=153 2023-07-22T12:34:21,085 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=820 B at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/session/20a6ddbb9c214d998c32b0bb0820f924 2023-07-22T12:34:21,090 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 20a6ddbb9c214d998c32b0bb0820f924 2023-07-22T12:34:21,090 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/meta/93cd26fa4bd94fc8b3a8b6de32bded79 as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/93cd26fa4bd94fc8b3a8b6de32bded79 2023-07-22T12:34:21,094 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/93cd26fa4bd94fc8b3a8b6de32bded79, entries=6, sequenceid=116, filesize=5.8 K 2023-07-22T12:34:21,095 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/session/20a6ddbb9c214d998c32b0bb0820f924 as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/20a6ddbb9c214d998c32b0bb0820f924 2023-07-22T12:34:21,099 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/20a6ddbb9c214d998c32b0bb0820f924, entries=2, sequenceid=116, filesize=5.5 K 2023-07-22T12:34:21,100 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.HRegion(3022): Finished flush of dataSize ~1.50 KB/1537, heapSize ~2.70 KB/2760, currentSize=0 B/0 for 697cc5ecf0962f5ab9f4126febf7ce33 in 834ms, sequenceid=116, compaction requested=true 2023-07-22T12:34:21,100 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.HRegion(2520): Flush status journal for 697cc5ecf0962f5ab9f4126febf7ce33: 2023-07-22T12:34:21,100 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. for snapshot_backup_system completed. 2023-07-22T12:34:21,100 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33.' region-info for snapshot=snapshot_backup_system 2023-07-22T12:34:21,100 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:34:21,100 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/b580ac8bece04b07a21605e555bdc696, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/5ea0158e2d574f758c68ee9c20c21191, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/a6d2695c5574440ea495f58a7c66ed03, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/8f78c6cced064299a30d72a0b4901528, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/606f117b1e384a54bb6dba5339e90e8f, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/15bf4f9b1b65467c96713e751468dc6c, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/b30f291cd15a4d92ac4b31f477e93428, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/3e46ffe2abec4ceba3ab7424eff1a910, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/e0ee030fbe0a48558b7d348517bf0f3c, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/93cd26fa4bd94fc8b3a8b6de32bded79] hfiles 2023-07-22T12:34:21,100 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (1/10): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/b580ac8bece04b07a21605e555bdc696 for snapshot=snapshot_backup_system 2023-07-22T12:34:21,100 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (2/10): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/5ea0158e2d574f758c68ee9c20c21191 for snapshot=snapshot_backup_system 2023-07-22T12:34:21,100 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (3/10): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/a6d2695c5574440ea495f58a7c66ed03 for snapshot=snapshot_backup_system 2023-07-22T12:34:21,100 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (4/10): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/8f78c6cced064299a30d72a0b4901528 for snapshot=snapshot_backup_system 2023-07-22T12:34:21,100 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (5/10): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/606f117b1e384a54bb6dba5339e90e8f for snapshot=snapshot_backup_system 2023-07-22T12:34:21,100 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (6/10): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/15bf4f9b1b65467c96713e751468dc6c for snapshot=snapshot_backup_system 2023-07-22T12:34:21,100 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (7/10): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/b30f291cd15a4d92ac4b31f477e93428 for snapshot=snapshot_backup_system 2023-07-22T12:34:21,100 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (8/10): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/3e46ffe2abec4ceba3ab7424eff1a910 for snapshot=snapshot_backup_system 2023-07-22T12:34:21,100 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (9/10): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/e0ee030fbe0a48558b7d348517bf0f3c for snapshot=snapshot_backup_system 2023-07-22T12:34:21,100 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (10/10): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/93cd26fa4bd94fc8b3a8b6de32bded79 for snapshot=snapshot_backup_system 2023-07-22T12:34:21,100 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/956010b3eaa64b4fa82ddea4557fac2c, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/0704967b71ce4a5cb0af553e6e47addb, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/62b5d2f1465b492ab802e26f5bf32f6b, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/c2fdeacbf4184719a11904adfeaf1fd8, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/660347589bb3465a82d9c9138151d6a4, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/2b4c40a0167b43f584d34520d18e9390, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/1550986ca81f443894a286cada73692c, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/a5d57e73d32d4eda8e52490df82775fa, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/83933de825724e92a18f2bfe93b821b6, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/82e877bb04384fa3a3f2266a337937ce, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/20a6ddbb9c214d998c32b0bb0820f924] hfiles 2023-07-22T12:34:21,100 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (1/11): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/956010b3eaa64b4fa82ddea4557fac2c for snapshot=snapshot_backup_system 2023-07-22T12:34:21,101 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (2/11): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/0704967b71ce4a5cb0af553e6e47addb for snapshot=snapshot_backup_system 2023-07-22T12:34:21,101 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (3/11): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/62b5d2f1465b492ab802e26f5bf32f6b for snapshot=snapshot_backup_system 2023-07-22T12:34:21,101 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (4/11): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/c2fdeacbf4184719a11904adfeaf1fd8 for snapshot=snapshot_backup_system 2023-07-22T12:34:21,101 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (5/11): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/660347589bb3465a82d9c9138151d6a4 for snapshot=snapshot_backup_system 2023-07-22T12:34:21,101 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (6/11): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/2b4c40a0167b43f584d34520d18e9390 for snapshot=snapshot_backup_system 2023-07-22T12:34:21,101 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (7/11): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/1550986ca81f443894a286cada73692c for snapshot=snapshot_backup_system 2023-07-22T12:34:21,101 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (8/11): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/a5d57e73d32d4eda8e52490df82775fa for snapshot=snapshot_backup_system 2023-07-22T12:34:21,101 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (9/11): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/83933de825724e92a18f2bfe93b821b6 for snapshot=snapshot_backup_system 2023-07-22T12:34:21,101 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (10/11): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/82e877bb04384fa3a3f2266a337937ce for snapshot=snapshot_backup_system 2023-07-22T12:34:21,101 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (11/11): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/20a6ddbb9c214d998c32b0bb0820f924 for snapshot=snapshot_backup_system 2023-07-22T12:34:21,505 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. 2023-07-22T12:34:21,506 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=154 2023-07-22T12:34:21,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster(3994): Remote procedure done, pid=154 2023-07-22T12:34:21,506 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 697cc5ecf0962f5ab9f4126febf7ce33 2023-07-22T12:34:21,507 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=154, ppid=153, state=RUNNABLE; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33 2023-07-22T12:34:21,509 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=154, resume processing ppid=153 2023-07-22T12:34:21,509 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:34:21,509 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=154, ppid=153, state=SUCCESS; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33 in 1.3950 sec 2023-07-22T12:34:21,510 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:34:21,510 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:34:21,510 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-07-22T12:34:21,511 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_backup_system 2023-07-22T12:34:21,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=153 2023-07-22T12:34:21,929 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:34:21,942 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:34:21,942 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_backup_system 2023-07-22T12:34:22,757 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:34:22,757 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 153 2023-07-22T12:34:22,758 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=153, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 3.0550 sec 2023-07-22T12:34:23,225 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_table2 2023-07-22T12:34:23,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=153 2023-07-22T12:34:23,810 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(115): Backup backup_1690029259664 started at 1690029263810. 2023-07-22T12:34:23,811 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(119): Backup session backup_1690029259664 has been started. 2023-07-22T12:34:23,817 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-07-22T12:34:23,818 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-07-22T12:34:23,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-07-22T12:34:23,818 INFO [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-07-22T12:34:23,818 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-07-22T12:34:23,819 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-07-22T12:34:23,819 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-07-22T12:34:23,820 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:34:23,821 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:23,821 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:34:23,821 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:34:23,821 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-07-22T12:34:23,821 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-07-22T12:34:23,822 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-07-22T12:34:23,822 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:34:23,822 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-07-22T12:34:23,822 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-07-22T12:34:23,822 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-07-22T12:34:23,822 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-07-22T12:34:23,822 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-07-22T12:34:23,823 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-07-22T12:34:23,823 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-07-22T12:34:23,823 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,43639,1690029056327' joining acquired barrier for procedure (rolllog) in zk 2023-07-22T12:34:23,825 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-07-22T12:34:23,825 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:23,825 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:34:23,825 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:34:23,826 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-07-22T12:34:23,826 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:34:23,826 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:34:23,826 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:34:23,826 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:23,827 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:34:23,827 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,43639,1690029056327' joining acquired barrier for procedure 'rolllog' on coordinator 2023-07-22T12:34:23,827 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@6ae04ba6[Count = 0] remaining members to acquire global barrier 2023-07-22T12:34:23,827 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-07-22T12:34:23,827 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-07-22T12:34:23,828 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-07-22T12:34:23,828 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-07-22T12:34:23,828 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-07-22T12:34:23,828 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-07-22T12:34:23,828 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-07-22T12:34:23,828 DEBUG [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:23,828 INFO [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1690029241461 highest: 1690029241461 on jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:23,828 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta:.meta(num 1690029241445) roll requested 2023-07-22T12:34:23,839 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029263828.meta, exclude list is [], retry=0 2023-07-22T12:34:23,843 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK] 2023-07-22T12:34:23,844 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029241445.meta with entries=24, filesize=5.77 KB; new WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029263828.meta 2023-07-22T12:34:23,844 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK]] 2023-07-22T12:34:23,844 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029241445.meta is not closed yet, will try archiving it next time 2023-07-22T12:34:23,844 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C43639%2C1690029056327:(num 1690029241461) roll requested 2023-07-22T12:34:23,856 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029263844, exclude list is [], retry=0 2023-07-22T12:34:23,858 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK] 2023-07-22T12:34:23,860 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029241461 with entries=15, filesize=4.05 KB; new WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029263844 2023-07-22T12:34:23,860 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK]] 2023-07-22T12:34:23,860 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029241461 is not closed yet, will try archiving it next time 2023-07-22T12:34:23,868 DEBUG [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-07-22T12:34:23,868 INFO [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1690029263844 on jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:23,876 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-07-22T12:34:23,876 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,43639,1690029056327' in zk 2023-07-22T12:34:23,882 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-07-22T12:34:23,882 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:34:23,882 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-07-22T12:34:23,882 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:23,882 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:34:23,882 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:34:23,883 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:34:23,883 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:34:23,883 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:34:23,883 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:23,884 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:34:23,884 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:34:23,884 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:23,885 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,43639,1690029056327': 2023-07-22T12:34:23,885 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,43639,1690029056327' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-07-22T12:34:23,885 INFO [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-07-22T12:34:23,885 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-07-22T12:34:23,885 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-07-22T12:34:23,885 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-07-22T12:34:23,885 INFO [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-07-22T12:34:23,886 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-07-22T12:34:23,886 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-07-22T12:34:23,886 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:34:23,886 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:34:23,886 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:34:23,887 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:34:23,887 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:34:23,887 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:34:23,887 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:34:23,887 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:34:23,887 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:34:23,888 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:34:23,888 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:23,888 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:34:23,888 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:34:23,889 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:23,892 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:34:23,892 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:34:23,892 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:34:23,892 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-07-22T12:34:23,893 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-07-22T12:34:23,893 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:34:23,893 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:34:23,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-07-22T12:34:23,996 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1690029263996_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } 2023-07-22T12:34:23,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029263996 (current time:1690029263996). 2023-07-22T12:34:23,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:34:23,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1690029263996_default_test-1690029065625 VERSION not specified, setting to 2 2023-07-22T12:34:24,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cf2d9ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:34:24,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:34:24,005 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:57450, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:34:24,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:34:24,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:34:24,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=155, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1690029263996_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } 2023-07-22T12:34:24,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1690029263996_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 }, snapshot procedure id = 155 2023-07-22T12:34:24,008 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1690029263996_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:34:24,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=155 2023-07-22T12:34:24,009 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1690029263996_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:34:24,010 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1690029263996_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:34:24,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=155 2023-07-22T12:34:24,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=155 2023-07-22T12:34:24,415 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1690029263996_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:34:24,415 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=156, ppid=155, state=RUNNABLE; SnapshotRegionProcedure baf4f0d945c66abea73108272203bc6e}] 2023-07-22T12:34:24,417 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=156, ppid=155, state=RUNNABLE; SnapshotRegionProcedure baf4f0d945c66abea73108272203bc6e 2023-07-22T12:34:24,478 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-07-22T12:34:24,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43639 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=156 2023-07-22T12:34:24,569 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1690029065625,,1690029166757.baf4f0d945c66abea73108272203bc6e. 2023-07-22T12:34:24,569 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] regionserver.HRegion(2520): Flush status journal for baf4f0d945c66abea73108272203bc6e: 2023-07-22T12:34:24,569 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1690029065625,,1690029166757.baf4f0d945c66abea73108272203bc6e. for snapshot_1690029263996_default_test-1690029065625 completed. 2023-07-22T12:34:24,569 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] snapshot.SnapshotManifest(240): Storing 'test-1690029065625,,1690029166757.baf4f0d945c66abea73108272203bc6e.' region-info for snapshot=snapshot_1690029263996_default_test-1690029065625 2023-07-22T12:34:24,569 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:34:24,569 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/baf4f0d945c66abea73108272203bc6e/f/28a90c4c400f435e9fc04a59e5431a5a_SeqId_4_] hfiles 2023-07-22T12:34:24,569 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/baf4f0d945c66abea73108272203bc6e/f/28a90c4c400f435e9fc04a59e5431a5a_SeqId_4_ for snapshot=snapshot_1690029263996_default_test-1690029065625 2023-07-22T12:34:24,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=155 2023-07-22T12:34:24,974 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1690029065625,,1690029166757.baf4f0d945c66abea73108272203bc6e. 2023-07-22T12:34:24,974 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=156 2023-07-22T12:34:24,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster(3994): Remote procedure done, pid=156 2023-07-22T12:34:24,975 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1690029263996_default_test-1690029065625 on region baf4f0d945c66abea73108272203bc6e 2023-07-22T12:34:24,976 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=156, ppid=155, state=RUNNABLE; SnapshotRegionProcedure baf4f0d945c66abea73108272203bc6e 2023-07-22T12:34:24,978 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=156, resume processing ppid=155 2023-07-22T12:34:24,978 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=156, ppid=155, state=SUCCESS; SnapshotRegionProcedure baf4f0d945c66abea73108272203bc6e in 562 msec 2023-07-22T12:34:24,978 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1690029263996_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:34:24,979 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1690029263996_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:34:24,979 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1690029263996_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:34:24,979 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1690029263996_default_test-1690029065625 2023-07-22T12:34:24,980 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_1690029263996_default_test-1690029065625 2023-07-22T12:34:25,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=155 2023-07-22T12:34:25,388 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1690029263996_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:34:25,392 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1690029263996_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:34:25,393 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_1690029263996_default_test-1690029065625 to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029263996_default_test-1690029065625 2023-07-22T12:34:26,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=155 2023-07-22T12:34:26,217 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1690029263996_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:34:26,217 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1690029263996_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 }, snapshot procedure id = 155 2023-07-22T12:34:26,218 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=155, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1690029263996_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } in 2.2110 sec 2023-07-22T12:34:28,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=155 2023-07-22T12:34:28,114 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1690029259664 2023-07-22T12:34:28,114 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-07-22T12:34:28,115 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-07-22T12:34:28,115 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1690029259664_test-1690029065625 2023-07-22T12:34:28,115 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1690029263996_default_test-1690029065625 to hdfs://localhost:34745/backupUT/backup_1690029259664/default/test-1690029065625/ 2023-07-22T12:34:28,115 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-07-22T12:34:28,137 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:34745, inputRoot=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947 2023-07-22T12:34:28,137 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,37923,1690029053050_-202120592_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:34745/backupUT/backup_1690029259664/default/test-1690029065625, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:34745/backupUT/backup_1690029259664/default/test-1690029065625/.hbase-snapshot/.tmp/snapshot_1690029263996_default_test-1690029065625 2023-07-22T12:34:28,137 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:34745, inputRoot=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947, snapshotDir=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029263996_default_test-1690029065625. 2023-07-22T12:34:28,151 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029263996_default_test-1690029065625 to hdfs://localhost:34745/backupUT/backup_1690029259664/default/test-1690029065625/.hbase-snapshot/.tmp/snapshot_1690029263996_default_test-1690029065625 2023-07-22T12:34:28,573 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:28,573 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:28,573 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:28,574 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:29,528 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/hadoop-7270152362444649456.jar 2023-07-22T12:34:29,528 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:29,528 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:29,529 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:29,529 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:29,530 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:29,530 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:29,530 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:29,531 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:29,531 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:29,532 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:34:29,532 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-07-22T12:34:29,533 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-07-22T12:34:29,533 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-07-22T12:34:29,533 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-07-22T12:34:29,534 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-07-22T12:34:29,534 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-07-22T12:34:29,534 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-07-22T12:34:29,535 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-07-22T12:34:29,535 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-07-22T12:34:29,535 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-07-22T12:34:29,536 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-07-22T12:34:29,536 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:34:29,537 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:34:29,537 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:34:29,538 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:34:29,538 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:34:29,538 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:34:29,539 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:34:29,579 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1690029263996_default_test-1690029065625' hfile list 2023-07-22T12:34:29,581 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-07-22T12:34:29,647 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029269615/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-07-22T12:34:29,647 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029269615/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-07-22T12:34:29,707 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-07-22T12:34:29,730 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/baf4f0d945c66abea73108272203bc6e/f/28a90c4c400f435e9fc04a59e5431a5a_SeqId_4_ output=hdfs://localhost:34745/backupUT/backup_1690029259664/default/test-1690029065625/archive/data/default/test-1690029065625/baf4f0d945c66abea73108272203bc6e/f/28a90c4c400f435e9fc04a59e5431a5a_SeqId_4_ 2023-07-22T12:34:29,730 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8724 (8.5 K) time=0sec 8.320M/sec 2023-07-22T12:34:30,680 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-07-22T12:34:30,681 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-07-22T12:34:30,686 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1690029263996_default_test-1690029065625 2023-07-22T12:34:30,686 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1690029263996_default_test-1690029065625 finished. 2023-07-22T12:34:30,686 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(1014): test-1690029065625 2023-07-22T12:34:30,692 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029259664 2023-07-22T12:34:30,692 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-07-22T12:34:31,096 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:34745/backupUT/backup_1690029259664/.backup.manifest 2023-07-22T12:34:31,096 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(358): Backup backup_1690029259664 finished: type=FULL,tablelist=test-1690029065625,targetRootDir=hdfs://localhost:34745/backupUT,startts=1690029263810,completets=1690029270692,bytescopied=0 2023-07-22T12:34:31,097 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-07-22T12:34:31,097 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1690029263996_default_test-1690029065625 2023-07-22T12:34:31,097 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1690029263996_default_test-1690029065625" 2023-07-22T12:34:31,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1690029263996_default_test-1690029065625 2023-07-22T12:34:31,101 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1690029263996_default_test-1690029065625 for backup backup_1690029259664 succeeded. 2023-07-22T12:34:31,102 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-07-22T12:34:31,105 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-07-22T12:34:31,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-07-22T12:34:31,107 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-07-22T12:34:31,109 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-07-22T12:34:31,112 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(376): Backup backup_1690029259664 completed. 2023-07-22T12:34:31,112 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:34:31,121 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] backup.TestFullRestore(265): backup complete 2023-07-22T12:34:31,121 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(101): HBase table table1 does not exist. It will be created during restore process 2023-07-22T12:34:31,122 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34745/backupUT/backup_1690029259664/.backup.manifest 2023-07-22T12:34:31,123 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029259664/.backup.manifest 2023-07-22T12:34:31,124 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34745/backupUT/backup_1690029259664/.backup.manifest 2023-07-22T12:34:31,125 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029259664/.backup.manifest 2023-07-22T12:34:31,125 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(148): Restoring 'faketable' to 'table1' from full backup image hdfs://localhost:34745/backupUT/backup_1690029259664/default/faketable 2023-07-22T12:34:31,142 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreSingleDNE Thread=602 (was 604), OpenFileDescriptor=686 (was 679) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=90 (was 96), ProcessCount=169 (was 169), AvailableMemoryMB=16216 (was 16232) 2023-07-22T12:34:31,142 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(130): Thread=602 is superior to 500 2023-07-22T12:34:31,156 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreSingleCommand Thread=602, OpenFileDescriptor=686, MaxFileDescriptor=60000, SystemLoadAverage=90, ProcessCount=169, AvailableMemoryMB=16216 2023-07-22T12:34:31,157 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(130): Thread=602 is superior to 500 2023-07-22T12:34:31,157 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] backup.TestFullRestore(76): test full restore on a single table empty table: command-line 2023-07-22T12:34:31,161 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49d32cfe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:34:31,163 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:34:31,164 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:44146, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:34:31,165 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:34:31,166 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:49614, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:34:31,176 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-07-22T12:34:31,181 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:34:31,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029271181 (current time:1690029271181). 2023-07-22T12:34:31,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:34:31,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-07-22T12:34:31,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143b6411, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:34:31,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:34:31,187 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:44148, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:34:31,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:34:31,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:34:31,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=157, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:34:31,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 157 2023-07-22T12:34:31,190 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:34:31,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=157 2023-07-22T12:34:31,191 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:34:31,192 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:34:31,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=157 2023-07-22T12:34:31,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=157 2023-07-22T12:34:31,597 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:34:31,597 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=158, ppid=157, state=RUNNABLE; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33}] 2023-07-22T12:34:31,599 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=158, ppid=157, state=RUNNABLE; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33 2023-07-22T12:34:31,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43639 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=158 2023-07-22T12:34:31,750 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. 2023-07-22T12:34:31,751 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.HRegion(2819): Flushing 697cc5ecf0962f5ab9f4126febf7ce33 2/2 column families, dataSize=1.14 KB heapSize=2.22 KB 2023-07-22T12:34:31,756 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/meta/d6830dd5c21a4377bee0840e832a1db8 is 169, key is trslm:hdfs://localhost:34745/backupUT\x00test-1690029065625/meta:log-roll-map/1690029270688/Put/seqid=0 2023-07-22T12:34:31,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=157 2023-07-22T12:34:32,160 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=451 B at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/meta/d6830dd5c21a4377bee0840e832a1db8 2023-07-22T12:34:32,166 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: d6830dd5c21a4377bee0840e832a1db8 2023-07-22T12:34:32,168 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/session/0438ad092ea24358b05211a8e4572521 is 452, key is session:backup_1690029259664/session:context/1690029271108/Put/seqid=0 2023-07-22T12:34:32,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=157 2023-07-22T12:34:32,572 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=721 B at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/session/0438ad092ea24358b05211a8e4572521 2023-07-22T12:34:32,577 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 0438ad092ea24358b05211a8e4572521 2023-07-22T12:34:32,578 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/meta/d6830dd5c21a4377bee0840e832a1db8 as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/d6830dd5c21a4377bee0840e832a1db8 2023-07-22T12:34:32,582 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/d6830dd5c21a4377bee0840e832a1db8, entries=4, sequenceid=127, filesize=5.5 K 2023-07-22T12:34:32,583 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/session/0438ad092ea24358b05211a8e4572521 as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/0438ad092ea24358b05211a8e4572521 2023-07-22T12:34:32,588 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/0438ad092ea24358b05211a8e4572521, entries=2, sequenceid=127, filesize=5.4 K 2023-07-22T12:34:32,588 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1172, heapSize ~2.19 KB/2240, currentSize=0 B/0 for 697cc5ecf0962f5ab9f4126febf7ce33 in 837ms, sequenceid=127, compaction requested=true 2023-07-22T12:34:32,589 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.HRegion(2520): Flush status journal for 697cc5ecf0962f5ab9f4126febf7ce33: 2023-07-22T12:34:32,589 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. for snapshot_backup_system completed. 2023-07-22T12:34:32,589 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33.' region-info for snapshot=snapshot_backup_system 2023-07-22T12:34:32,589 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:34:32,589 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/b580ac8bece04b07a21605e555bdc696, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/5ea0158e2d574f758c68ee9c20c21191, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/a6d2695c5574440ea495f58a7c66ed03, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/8f78c6cced064299a30d72a0b4901528, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/606f117b1e384a54bb6dba5339e90e8f, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/15bf4f9b1b65467c96713e751468dc6c, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/b30f291cd15a4d92ac4b31f477e93428, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/3e46ffe2abec4ceba3ab7424eff1a910, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/e0ee030fbe0a48558b7d348517bf0f3c, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/93cd26fa4bd94fc8b3a8b6de32bded79, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/d6830dd5c21a4377bee0840e832a1db8] hfiles 2023-07-22T12:34:32,589 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (1/11): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/b580ac8bece04b07a21605e555bdc696 for snapshot=snapshot_backup_system 2023-07-22T12:34:32,589 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (2/11): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/5ea0158e2d574f758c68ee9c20c21191 for snapshot=snapshot_backup_system 2023-07-22T12:34:32,589 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (3/11): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/a6d2695c5574440ea495f58a7c66ed03 for snapshot=snapshot_backup_system 2023-07-22T12:34:32,589 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (4/11): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/8f78c6cced064299a30d72a0b4901528 for snapshot=snapshot_backup_system 2023-07-22T12:34:32,589 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (5/11): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/606f117b1e384a54bb6dba5339e90e8f for snapshot=snapshot_backup_system 2023-07-22T12:34:32,589 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (6/11): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/15bf4f9b1b65467c96713e751468dc6c for snapshot=snapshot_backup_system 2023-07-22T12:34:32,589 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (7/11): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/b30f291cd15a4d92ac4b31f477e93428 for snapshot=snapshot_backup_system 2023-07-22T12:34:32,589 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (8/11): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/3e46ffe2abec4ceba3ab7424eff1a910 for snapshot=snapshot_backup_system 2023-07-22T12:34:32,589 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (9/11): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/e0ee030fbe0a48558b7d348517bf0f3c for snapshot=snapshot_backup_system 2023-07-22T12:34:32,589 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (10/11): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/93cd26fa4bd94fc8b3a8b6de32bded79 for snapshot=snapshot_backup_system 2023-07-22T12:34:32,589 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (11/11): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/d6830dd5c21a4377bee0840e832a1db8 for snapshot=snapshot_backup_system 2023-07-22T12:34:32,589 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/956010b3eaa64b4fa82ddea4557fac2c, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/0704967b71ce4a5cb0af553e6e47addb, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/62b5d2f1465b492ab802e26f5bf32f6b, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/c2fdeacbf4184719a11904adfeaf1fd8, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/660347589bb3465a82d9c9138151d6a4, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/2b4c40a0167b43f584d34520d18e9390, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/1550986ca81f443894a286cada73692c, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/a5d57e73d32d4eda8e52490df82775fa, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/83933de825724e92a18f2bfe93b821b6, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/82e877bb04384fa3a3f2266a337937ce, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/20a6ddbb9c214d998c32b0bb0820f924, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/0438ad092ea24358b05211a8e4572521] hfiles 2023-07-22T12:34:32,589 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (1/12): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/956010b3eaa64b4fa82ddea4557fac2c for snapshot=snapshot_backup_system 2023-07-22T12:34:32,589 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (2/12): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/0704967b71ce4a5cb0af553e6e47addb for snapshot=snapshot_backup_system 2023-07-22T12:34:32,589 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (3/12): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/62b5d2f1465b492ab802e26f5bf32f6b for snapshot=snapshot_backup_system 2023-07-22T12:34:32,589 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (4/12): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/c2fdeacbf4184719a11904adfeaf1fd8 for snapshot=snapshot_backup_system 2023-07-22T12:34:32,589 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (5/12): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/660347589bb3465a82d9c9138151d6a4 for snapshot=snapshot_backup_system 2023-07-22T12:34:32,589 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (6/12): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/2b4c40a0167b43f584d34520d18e9390 for snapshot=snapshot_backup_system 2023-07-22T12:34:32,589 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (7/12): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/1550986ca81f443894a286cada73692c for snapshot=snapshot_backup_system 2023-07-22T12:34:32,589 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (8/12): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/a5d57e73d32d4eda8e52490df82775fa for snapshot=snapshot_backup_system 2023-07-22T12:34:32,589 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (9/12): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/83933de825724e92a18f2bfe93b821b6 for snapshot=snapshot_backup_system 2023-07-22T12:34:32,589 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (10/12): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/82e877bb04384fa3a3f2266a337937ce for snapshot=snapshot_backup_system 2023-07-22T12:34:32,589 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (11/12): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/20a6ddbb9c214d998c32b0bb0820f924 for snapshot=snapshot_backup_system 2023-07-22T12:34:32,590 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (12/12): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/0438ad092ea24358b05211a8e4572521 for snapshot=snapshot_backup_system 2023-07-22T12:34:32,999 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. 2023-07-22T12:34:32,999 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=158 2023-07-22T12:34:32,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster(3994): Remote procedure done, pid=158 2023-07-22T12:34:32,999 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 697cc5ecf0962f5ab9f4126febf7ce33 2023-07-22T12:34:33,000 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=158, ppid=157, state=RUNNABLE; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33 2023-07-22T12:34:33,002 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=158, resume processing ppid=157 2023-07-22T12:34:33,002 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:34:33,002 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=158, ppid=157, state=SUCCESS; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33 in 1.4040 sec 2023-07-22T12:34:33,003 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:34:33,003 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:34:33,003 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-07-22T12:34:33,005 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_backup_system 2023-07-22T12:34:33,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=157 2023-07-22T12:34:33,413 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:34:33,429 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:34:33,429 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_backup_system 2023-07-22T12:34:34,246 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:34:34,246 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 157 2023-07-22T12:34:34,247 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=157, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 3.0580 sec 2023-07-22T12:34:35,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=157 2023-07-22T12:34:35,295 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(115): Backup backup_1690029271161 started at 1690029275295. 2023-07-22T12:34:35,297 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(119): Backup session backup_1690029271161 has been started. 2023-07-22T12:34:35,301 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-07-22T12:34:35,301 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-07-22T12:34:35,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure.ProcedureCoordinator(143): Procedure rolllog was in running list but was completed. Accepting new attempt. 2023-07-22T12:34:35,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-07-22T12:34:35,302 INFO [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-07-22T12:34:35,302 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-07-22T12:34:35,303 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-07-22T12:34:35,303 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-07-22T12:34:35,303 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:34:35,305 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:35,305 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:34:35,305 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:34:35,306 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-07-22T12:34:35,306 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-07-22T12:34:35,306 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-07-22T12:34:35,306 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:34:35,306 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-07-22T12:34:35,307 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-07-22T12:34:35,307 WARN [zk-event-processor-pool-0 {}] procedure.ProcedureMember(133): A completed old subproc rolllog is still present, removing 2023-07-22T12:34:35,307 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-07-22T12:34:35,307 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-07-22T12:34:35,307 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-07-22T12:34:35,307 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-07-22T12:34:35,307 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-07-22T12:34:35,307 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,43639,1690029056327' joining acquired barrier for procedure (rolllog) in zk 2023-07-22T12:34:35,310 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-07-22T12:34:35,310 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:35,310 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:34:35,310 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:34:35,310 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-07-22T12:34:35,310 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:34:35,310 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:34:35,312 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:34:35,313 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:35,313 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:34:35,313 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,43639,1690029056327' joining acquired barrier for procedure 'rolllog' on coordinator 2023-07-22T12:34:35,313 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@7f6e52e1[Count = 0] remaining members to acquire global barrier 2023-07-22T12:34:35,313 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-07-22T12:34:35,313 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-07-22T12:34:35,315 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-07-22T12:34:35,315 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-07-22T12:34:35,315 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-07-22T12:34:35,315 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-07-22T12:34:35,315 DEBUG [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:35,315 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-07-22T12:34:35,315 INFO [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1690029263844 highest: 1690029263844 on jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:35,315 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta:.meta(num 1690029263828) roll requested 2023-07-22T12:34:35,332 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029275315.meta, exclude list is [], retry=0 2023-07-22T12:34:35,336 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK] 2023-07-22T12:34:35,337 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029263828.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029275315.meta 2023-07-22T12:34:35,337 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK]] 2023-07-22T12:34:35,337 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029263828.meta is not closed yet, will try archiving it next time 2023-07-22T12:34:35,337 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C43639%2C1690029056327:(num 1690029263844) roll requested 2023-07-22T12:34:35,340 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029263828.meta to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/oldWALs/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029263828.meta 2023-07-22T12:34:35,351 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029275338, exclude list is [], retry=0 2023-07-22T12:34:35,355 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK] 2023-07-22T12:34:35,356 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029263844 with entries=10, filesize=2.31 KB; new WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029275338 2023-07-22T12:34:35,357 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK]] 2023-07-22T12:34:35,357 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029263844 is not closed yet, will try archiving it next time 2023-07-22T12:34:35,357 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029241461 to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/oldWALs/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029241461 2023-07-22T12:34:35,376 DEBUG [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 61 2023-07-22T12:34:35,376 INFO [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1690029275338 on jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:35,396 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-07-22T12:34:35,396 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,43639,1690029056327' in zk 2023-07-22T12:34:35,401 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-07-22T12:34:35,401 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:34:35,401 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-07-22T12:34:35,402 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:35,402 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:34:35,402 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:34:35,402 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:34:35,402 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:34:35,403 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:34:35,403 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:35,403 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:34:35,404 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:34:35,404 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:35,404 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,43639,1690029056327': 2023-07-22T12:34:35,404 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,43639,1690029056327' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-07-22T12:34:35,404 INFO [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-07-22T12:34:35,404 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-07-22T12:34:35,404 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-07-22T12:34:35,405 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-07-22T12:34:35,405 INFO [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-07-22T12:34:35,407 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-07-22T12:34:35,407 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-07-22T12:34:35,407 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:34:35,407 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:34:35,407 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:34:35,407 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:34:35,407 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:34:35,407 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:34:35,408 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:34:35,408 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:34:35,408 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:34:35,408 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:34:35,409 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:35,409 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:34:35,409 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:34:35,410 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:35,419 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:34:35,419 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:34:35,419 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:34:35,419 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-07-22T12:34:35,419 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-07-22T12:34:35,420 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:34:35,421 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:34:35,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-07-22T12:34:35,524 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1690029275523_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } 2023-07-22T12:34:35,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029275524 (current time:1690029275524). 2023-07-22T12:34:35,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:34:35,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1690029275523_default_test-1690029065625 VERSION not specified, setting to 2 2023-07-22T12:34:35,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19414fca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:34:35,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:34:35,533 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:44158, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:34:35,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:34:35,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:34:35,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=159, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1690029275523_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } 2023-07-22T12:34:35,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1690029275523_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 }, snapshot procedure id = 159 2023-07-22T12:34:35,539 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1690029275523_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:34:35,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=159 2023-07-22T12:34:35,540 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1690029275523_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:34:35,541 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1690029275523_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:34:35,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=159 2023-07-22T12:34:35,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=159 2023-07-22T12:34:35,948 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1690029275523_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:34:35,949 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=160, ppid=159, state=RUNNABLE; SnapshotRegionProcedure baf4f0d945c66abea73108272203bc6e}] 2023-07-22T12:34:35,950 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=160, ppid=159, state=RUNNABLE; SnapshotRegionProcedure baf4f0d945c66abea73108272203bc6e 2023-07-22T12:34:36,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43639 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=160 2023-07-22T12:34:36,102 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1690029065625,,1690029166757.baf4f0d945c66abea73108272203bc6e. 2023-07-22T12:34:36,102 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] regionserver.HRegion(2520): Flush status journal for baf4f0d945c66abea73108272203bc6e: 2023-07-22T12:34:36,102 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1690029065625,,1690029166757.baf4f0d945c66abea73108272203bc6e. for snapshot_1690029275523_default_test-1690029065625 completed. 2023-07-22T12:34:36,102 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] snapshot.SnapshotManifest(240): Storing 'test-1690029065625,,1690029166757.baf4f0d945c66abea73108272203bc6e.' region-info for snapshot=snapshot_1690029275523_default_test-1690029065625 2023-07-22T12:34:36,102 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:34:36,103 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/baf4f0d945c66abea73108272203bc6e/f/28a90c4c400f435e9fc04a59e5431a5a_SeqId_4_] hfiles 2023-07-22T12:34:36,103 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/baf4f0d945c66abea73108272203bc6e/f/28a90c4c400f435e9fc04a59e5431a5a_SeqId_4_ for snapshot=snapshot_1690029275523_default_test-1690029065625 2023-07-22T12:34:36,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=159 2023-07-22T12:34:36,508 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1690029065625,,1690029166757.baf4f0d945c66abea73108272203bc6e. 2023-07-22T12:34:36,508 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=160 2023-07-22T12:34:36,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster(3994): Remote procedure done, pid=160 2023-07-22T12:34:36,508 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1690029275523_default_test-1690029065625 on region baf4f0d945c66abea73108272203bc6e 2023-07-22T12:34:36,509 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=160, ppid=159, state=RUNNABLE; SnapshotRegionProcedure baf4f0d945c66abea73108272203bc6e 2023-07-22T12:34:36,511 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=160, resume processing ppid=159 2023-07-22T12:34:36,511 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=160, ppid=159, state=SUCCESS; SnapshotRegionProcedure baf4f0d945c66abea73108272203bc6e in 561 msec 2023-07-22T12:34:36,511 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1690029275523_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:34:36,512 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1690029275523_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:34:36,512 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1690029275523_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:34:36,512 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1690029275523_default_test-1690029065625 2023-07-22T12:34:36,514 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_1690029275523_default_test-1690029065625 2023-07-22T12:34:36,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=159 2023-07-22T12:34:36,923 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1690029275523_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:34:36,928 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1690029275523_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:34:36,928 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_1690029275523_default_test-1690029065625 to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029275523_default_test-1690029065625 2023-07-22T12:34:37,342 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1690029275523_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:34:37,342 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1690029275523_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 }, snapshot procedure id = 159 2023-07-22T12:34:37,343 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=159, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1690029275523_default_test-1690029065625 table=test-1690029065625 type=FLUSH ttl=0 } in 1.8060 sec 2023-07-22T12:34:37,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=159 2023-07-22T12:34:37,644 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1690029271161 2023-07-22T12:34:37,644 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-07-22T12:34:37,644 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-07-22T12:34:37,644 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1690029271161_test-1690029065625 2023-07-22T12:34:37,644 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1690029275523_default_test-1690029065625 to hdfs://localhost:34745/backupUT/backup_1690029271161/default/test-1690029065625/ 2023-07-22T12:34:37,644 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-07-22T12:34:37,667 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:34745, inputRoot=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947 2023-07-22T12:34:37,667 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,37923,1690029053050_1832548305_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:34745/backupUT/backup_1690029271161/default/test-1690029065625, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:34745/backupUT/backup_1690029271161/default/test-1690029065625/.hbase-snapshot/.tmp/snapshot_1690029275523_default_test-1690029065625 2023-07-22T12:34:37,667 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:34745, inputRoot=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947, snapshotDir=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029275523_default_test-1690029065625. 2023-07-22T12:34:37,675 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029275523_default_test-1690029065625 to hdfs://localhost:34745/backupUT/backup_1690029271161/default/test-1690029065625/.hbase-snapshot/.tmp/snapshot_1690029275523_default_test-1690029065625 2023-07-22T12:34:38,107 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:38,107 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:38,108 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:38,108 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:39,073 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/hadoop-9187008123651984917.jar 2023-07-22T12:34:39,073 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:39,073 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:39,074 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:39,074 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:39,075 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:39,075 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:39,075 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:39,075 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:39,076 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:39,076 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:34:39,077 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-07-22T12:34:39,077 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-07-22T12:34:39,078 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-07-22T12:34:39,078 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-07-22T12:34:39,078 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-07-22T12:34:39,079 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-07-22T12:34:39,079 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-07-22T12:34:39,079 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-07-22T12:34:39,080 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-07-22T12:34:39,080 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-07-22T12:34:39,080 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-07-22T12:34:39,081 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:34:39,081 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:34:39,082 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:34:39,082 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:34:39,082 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:34:39,083 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:34:39,083 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:34:39,124 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1690029275523_default_test-1690029065625' hfile list 2023-07-22T12:34:39,126 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-07-22T12:34:39,192 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029279159/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-07-22T12:34:39,192 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029279159/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-07-22T12:34:39,245 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-07-22T12:34:39,270 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/baf4f0d945c66abea73108272203bc6e/f/28a90c4c400f435e9fc04a59e5431a5a_SeqId_4_ output=hdfs://localhost:34745/backupUT/backup_1690029271161/default/test-1690029065625/archive/data/default/test-1690029065625/baf4f0d945c66abea73108272203bc6e/f/28a90c4c400f435e9fc04a59e5431a5a_SeqId_4_ 2023-07-22T12:34:39,270 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8724 (8.5 K) time=0sec 4.160M/sec 2023-07-22T12:34:40,227 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-07-22T12:34:40,228 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-07-22T12:34:40,234 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1690029275523_default_test-1690029065625 2023-07-22T12:34:40,234 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1690029275523_default_test-1690029065625 finished. 2023-07-22T12:34:40,234 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(1014): test-1690029065625 2023-07-22T12:34:40,240 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029271161 2023-07-22T12:34:40,240 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-07-22T12:34:40,645 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:34745/backupUT/backup_1690029271161/.backup.manifest 2023-07-22T12:34:40,645 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(358): Backup backup_1690029271161 finished: type=FULL,tablelist=test-1690029065625,targetRootDir=hdfs://localhost:34745/backupUT,startts=1690029275295,completets=1690029280240,bytescopied=0 2023-07-22T12:34:40,645 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-07-22T12:34:40,645 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1690029275523_default_test-1690029065625 2023-07-22T12:34:40,646 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1690029275523_default_test-1690029065625" 2023-07-22T12:34:40,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1690029275523_default_test-1690029065625 2023-07-22T12:34:40,650 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1690029275523_default_test-1690029065625 for backup backup_1690029271161 succeeded. 2023-07-22T12:34:40,650 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-07-22T12:34:40,654 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-07-22T12:34:40,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-07-22T12:34:40,656 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-07-22T12:34:40,657 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-07-22T12:34:40,662 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(376): Backup backup_1690029271161 completed. 2023-07-22T12:34:40,662 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:34:40,662 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] backup.TestFullRestore(80): backup complete 2023-07-22T12:34:40,676 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e06afd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:34:40,677 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:34:40,679 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:43140, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:34:40,679 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(101): HBase table table1 does not exist. It will be created during restore process 2023-07-22T12:34:40,680 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34745/backupUT/backup_1690029271161/.backup.manifest 2023-07-22T12:34:40,681 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029271161/.backup.manifest 2023-07-22T12:34:40,682 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34745/backupUT/backup_1690029271161/.backup.manifest 2023-07-22T12:34:40,683 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029271161/.backup.manifest 2023-07-22T12:34:40,683 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(148): Restoring 'test-1690029065625' to 'table1' from full backup image hdfs://localhost:34745/backupUT/backup_1690029271161/default/test-1690029065625 2023-07-22T12:34:40,689 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] util.RestoreTool(487): Creating target table 'table1' 2023-07-22T12:34:40,689 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] util.RestoreTool(391): Parsing region dir: hdfs://localhost:34745/backupUT/backup_1690029271161/default/test-1690029065625/archive/data/default/test-1690029065625/baf4f0d945c66abea73108272203bc6e 2023-07-22T12:34:40,690 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:34745/backupUT/backup_1690029271161/default/test-1690029065625/archive/data/default/test-1690029065625/baf4f0d945c66abea73108272203bc6e/f in region [hdfs://localhost:34745/backupUT/backup_1690029271161/default/test-1690029065625/archive/data/default/test-1690029065625/baf4f0d945c66abea73108272203bc6e] 2023-07-22T12:34:40,692 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] util.RestoreTool(442): Trying to figure out region boundaries hfile=hdfs://localhost:34745/backupUT/backup_1690029271161/default/test-1690029065625/archive/data/default/test-1690029065625/baf4f0d945c66abea73108272203bc6e/f/28a90c4c400f435e9fc04a59e5431a5a_SeqId_4_ first=row0 last=row98 2023-07-22T12:34:40,692 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 28a90c4c400f435e9fc04a59e5431a5a_SeqId_4_ 2023-07-22T12:34:40,693 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:34:40,694 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:35474, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:34:40,695 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:34:40,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=161, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=table1 2023-07-22T12:34:40,698 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=161, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:34:40,698 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "table1" procId is: 161 2023-07-22T12:34:40,698 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=161, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:34:40,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=161 2023-07-22T12:34:40,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=161 2023-07-22T12:34:41,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=161 2023-07-22T12:34:41,104 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => de587539e56a809016723f0ce4633348, NAME => 'table1,,1690029280695.de587539e56a809016723f0ce4633348.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947 2023-07-22T12:34:41,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=161 2023-07-22T12:34:41,510 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(888): Instantiated table1,,1690029280695.de587539e56a809016723f0ce4633348.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:34:41,510 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1668): Closing de587539e56a809016723f0ce4633348, disabling compactions & flushes 2023-07-22T12:34:41,510 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1690): Closing region table1,,1690029280695.de587539e56a809016723f0ce4633348. 2023-07-22T12:34:41,510 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on table1,,1690029280695.de587539e56a809016723f0ce4633348. 2023-07-22T12:34:41,510 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on table1,,1690029280695.de587539e56a809016723f0ce4633348. after waiting 0 ms 2023-07-22T12:34:41,510 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region table1,,1690029280695.de587539e56a809016723f0ce4633348. 2023-07-22T12:34:41,510 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1909): Closed table1,,1690029280695.de587539e56a809016723f0ce4633348. 2023-07-22T12:34:41,510 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1622): Region close journal for de587539e56a809016723f0ce4633348: 2023-07-22T12:34:41,511 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=161, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:34:41,511 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"table1,,1690029280695.de587539e56a809016723f0ce4633348.","families":{"info":[{"qualifier":"regioninfo","vlen":40,"tag":[],"timestamp":"1690029281511"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690029281511"}]},"ts":"1690029281511"} 2023-07-22T12:34:41,512 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:34:41,513 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=161, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:34:41,513 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029281513"}]},"ts":"1690029281513"} 2023-07-22T12:34:41,514 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=ENABLING in hbase:meta 2023-07-22T12:34:41,519 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=table1, region=de587539e56a809016723f0ce4633348, ASSIGN}] 2023-07-22T12:34:41,521 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=table1, region=de587539e56a809016723f0ce4633348, ASSIGN 2023-07-22T12:34:41,521 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=table1, region=de587539e56a809016723f0ce4633348, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,43639,1690029056327; forceNewPlan=false, retain=false 2023-07-22T12:34:41,674 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=162 updating hbase:meta row=de587539e56a809016723f0ce4633348, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:41,678 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE; OpenRegionProcedure de587539e56a809016723f0ce4633348, server=jenkins-hbase3.apache.org,43639,1690029056327}] 2023-07-22T12:34:41,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=161 2023-07-22T12:34:41,832 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] handler.AssignRegionHandler(130): Open table1,,1690029280695.de587539e56a809016723f0ce4633348. 2023-07-22T12:34:41,833 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(7254): Opening region: {ENCODED => de587539e56a809016723f0ce4633348, NAME => 'table1,,1690029280695.de587539e56a809016723f0ce4633348.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:34:41,833 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:34:41,833 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 de587539e56a809016723f0ce4633348 2023-07-22T12:34:41,833 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(888): Instantiated table1,,1690029280695.de587539e56a809016723f0ce4633348.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:34:41,833 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(7296): checking encryption for de587539e56a809016723f0ce4633348 2023-07-22T12:34:41,833 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(7299): checking classloading for de587539e56a809016723f0ce4633348 2023-07-22T12:34:41,834 INFO [StoreOpener-de587539e56a809016723f0ce4633348-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region de587539e56a809016723f0ce4633348 2023-07-22T12:34:41,836 INFO [StoreOpener-de587539e56a809016723f0ce4633348-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de587539e56a809016723f0ce4633348 columnFamilyName f 2023-07-22T12:34:41,836 DEBUG [StoreOpener-de587539e56a809016723f0ce4633348-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:34:41,836 INFO [StoreOpener-de587539e56a809016723f0ce4633348-1 {}] regionserver.HStore(324): Store=de587539e56a809016723f0ce4633348/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:34:41,837 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/table1/de587539e56a809016723f0ce4633348 2023-07-22T12:34:41,837 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/table1/de587539e56a809016723f0ce4633348 2023-07-22T12:34:41,840 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(1076): writing seq id for de587539e56a809016723f0ce4633348 2023-07-22T12:34:41,842 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/table1/de587539e56a809016723f0ce4633348/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:34:41,842 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(1093): Opened de587539e56a809016723f0ce4633348; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10689059520, jitterRate=-0.004503756761550903}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-07-22T12:34:41,843 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(992): Region open journal for de587539e56a809016723f0ce4633348: 2023-07-22T12:34:41,843 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegionServer(2552): Post open deploy tasks for table1,,1690029280695.de587539e56a809016723f0ce4633348., pid=163, masterSystemTime=1690029281829 2023-07-22T12:34:41,845 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegionServer(2579): Finished post open deploy task for table1,,1690029280695.de587539e56a809016723f0ce4633348. 2023-07-22T12:34:41,845 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] handler.AssignRegionHandler(158): Opened table1,,1690029280695.de587539e56a809016723f0ce4633348. 2023-07-22T12:34:41,845 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=162 updating hbase:meta row=de587539e56a809016723f0ce4633348, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:41,848 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=163, resume processing ppid=162 2023-07-22T12:34:41,848 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=163, ppid=162, state=SUCCESS; OpenRegionProcedure de587539e56a809016723f0ce4633348, server=jenkins-hbase3.apache.org,43639,1690029056327 in 170 msec 2023-07-22T12:34:41,849 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=162, resume processing ppid=161 2023-07-22T12:34:41,850 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=162, ppid=161, state=SUCCESS; TransitRegionStateProcedure table=table1, region=de587539e56a809016723f0ce4633348, ASSIGN in 329 msec 2023-07-22T12:34:41,850 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=161, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:34:41,850 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029281850"}]},"ts":"1690029281850"} 2023-07-22T12:34:41,851 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=ENABLED in hbase:meta 2023-07-22T12:34:41,853 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=161, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:34:41,854 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=161, state=SUCCESS; CreateTableProcedure table=table1 in 1.1580 sec 2023-07-22T12:34:42,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=161 2023-07-22T12:34:42,807 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:34745/backupUT/backup_1690029271161/default/test-1690029065625/archive/data/default/test-1690029065625/baf4f0d945c66abea73108272203bc6e from hbase tables test-1690029065625 to tables table1 2023-07-22T12:34:42,807 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceRestoreJob(72): Restore test-1690029065625 into table1 2023-07-22T12:34:42,809 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-default-table1-1690029282807 from hdfs://localhost:34745/backupUT/backup_1690029271161/default/test-1690029065625/archive/data/default/test-1690029065625/baf4f0d945c66abea73108272203bc6e 2023-07-22T12:34:42,814 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43b0b87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:34:42,814 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:34:42,816 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:35484, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:34:42,817 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-07-22T12:34:42,817 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table table1 2023-07-22T12:34:42,819 DEBUG [hconnection-0xad8d3a6-shared-pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:34:42,820 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:43152, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:34:42,821 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for table1: 2023-07-22T12:34:42,821 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-07-22T12:34:42,821 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/partitions_5224a524-1370-4357-afc9-c7816a888cab 2023-07-22T12:34:43,225 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_table1 2023-07-22T12:34:43,225 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_table1 Metrics about Tables on a single HBase RegionServer 2023-07-22T12:34:43,229 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:43,229 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:43,229 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:43,230 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:44,201 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/hadoop-5578688075829965071.jar 2023-07-22T12:34:44,202 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:44,202 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:44,203 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:44,203 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:44,203 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:44,204 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:44,204 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:44,204 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:44,204 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:44,205 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:34:44,206 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-07-22T12:34:44,206 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-07-22T12:34:44,206 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-07-22T12:34:44,207 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-07-22T12:34:44,207 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-07-22T12:34:44,208 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-07-22T12:34:44,208 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-07-22T12:34:44,208 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-07-22T12:34:44,209 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-07-22T12:34:44,209 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-07-22T12:34:44,209 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-07-22T12:34:44,210 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:44,210 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:44,210 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:44,211 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:44,211 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:44,211 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:44,212 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:34:44,212 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: table1 2023-07-22T12:34:44,212 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:34:44,212 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-07-22T12:34:44,212 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-07-22T12:34:44,213 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:34:44,224 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-07-22T12:34:44,298 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029284264/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-07-22T12:34:44,298 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029284264/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-07-22T12:34:44,347 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:34745/backupUT/backup_1690029271161/default/test-1690029065625/archive/data/default/test-1690029065625/baf4f0d945c66abea73108272203bc6e/f/28a90c4c400f435e9fc04a59e5431a5a_SeqId_4_ 2023-07-22T12:34:44,350 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-07-22T12:34:44,352 DEBUG [LocalJobRunner Map Task Executor #0 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 28a90c4c400f435e9fc04a59e5431a5a_SeqId_4_ 2023-07-22T12:34:44,370 DEBUG [pool-533-thread-1 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ce8da2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:34:44,371 DEBUG [hconnection-0x1a787f60-metaLookup-shared--pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:34:44,373 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:43156, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:34:44,376 DEBUG [pool-533-thread-1 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:34:44,376 DEBUG [pool-533-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-07-22T12:34:44,376 DEBUG [pool-533-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-07-22T12:34:44,380 DEBUG [pool-533-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-default-table1-1690029282807/_temporary/0/_temporary/attempt_local1188970389_0025_r_000000_0/f/30ee8bf667464b4cbaddca0c642236fc is 37, key is row10/f:q1/1690029068268/Put/seqid=0 2023-07-22T12:34:44,381 WARN [Thread-2579 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:43639] Specified favored nodes: [jenkins-hbase3.apache.org:43639] 2023-07-22T12:34:45,346 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-default-table1-1690029282807 2023-07-22T12:34:45,351 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40ea4886, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:34:45,352 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:34:45,354 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:43172, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:34:45,358 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-default-table1-1690029282807/_SUCCESS 2023-07-22T12:34:45,361 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:34:45,363 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:35488, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:34:45,371 DEBUG [LoadIncrementalHFiles-0 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 30ee8bf667464b4cbaddca0c642236fc 2023-07-22T12:34:45,371 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-default-table1-1690029282807/f/30ee8bf667464b4cbaddca0c642236fc first=Optional[row0] last=Optional[row98] 2023-07-22T12:34:45,372 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=table1,,1690029280695.de587539e56a809016723f0ce4633348., hostname=jenkins-hbase3.apache.org,43639,1690029056327, seqNum=2 for row with hfile group [{f,hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-default-table1-1690029282807/f/30ee8bf667464b4cbaddca0c642236fc}] 2023-07-22T12:34:45,377 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-default-table1-1690029282807/f/30ee8bf667464b4cbaddca0c642236fc for inclusion in de587539e56a809016723f0ce4633348/f 2023-07-22T12:34:45,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-07-22T12:34:45,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HStore(625): Region bounds: first= last= 2023-07-22T12:34:45,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 30ee8bf667464b4cbaddca0c642236fc 2023-07-22T12:34:45,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HRegion(2520): Flush status journal for de587539e56a809016723f0ce4633348: 2023-07-22T12:34:45,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/hbase-staging/bulk_output-default-table1-1690029282807/f/30ee8bf667464b4cbaddca0c642236fc to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/staging/jenkins__table1__naub5qv38vim7b8qrql7hmkrmupin4keama53ouk6ke0a7pk55ct4vgnvbh9iblr/f/30ee8bf667464b4cbaddca0c642236fc 2023-07-22T12:34:45,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/staging/jenkins__table1__naub5qv38vim7b8qrql7hmkrmupin4keama53ouk6ke0a7pk55ct4vgnvbh9iblr/f/30ee8bf667464b4cbaddca0c642236fc as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/table1/de587539e56a809016723f0ce4633348/f/1342e9345f484cc8a1dd24c0d41f72c9_SeqId_4_ 2023-07-22T12:34:45,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d9d1851, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:34:45,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:34:45,388 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:35492, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:34:45,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:34:45,394 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:43176, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:34:45,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:34:45,405 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/staging/jenkins__table1__naub5qv38vim7b8qrql7hmkrmupin4keama53ouk6ke0a7pk55ct4vgnvbh9iblr/f/30ee8bf667464b4cbaddca0c642236fc into de587539e56a809016723f0ce4633348/f as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/table1/de587539e56a809016723f0ce4633348/f/1342e9345f484cc8a1dd24c0d41f72c9_SeqId_4_ - updating store file list. 2023-07-22T12:34:45,409 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/table1/de587539e56a809016723f0ce4633348/f/1342e9345f484cc8a1dd24c0d41f72c9_SeqId_4_ into de587539e56a809016723f0ce4633348/f 2023-07-22T12:34:45,409 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/staging/jenkins__table1__naub5qv38vim7b8qrql7hmkrmupin4keama53ouk6ke0a7pk55ct4vgnvbh9iblr/f/30ee8bf667464b4cbaddca0c642236fc into de587539e56a809016723f0ce4633348/f (new location: hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/table1/de587539e56a809016723f0ce4633348/f/1342e9345f484cc8a1dd24c0d41f72c9_SeqId_4_) 2023-07-22T12:34:45,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/staging/jenkins__table1__naub5qv38vim7b8qrql7hmkrmupin4keama53ouk6ke0a7pk55ct4vgnvbh9iblr/f/30ee8bf667464b4cbaddca0c642236fc 2023-07-22T12:34:45,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f909457, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:34:45,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:34:45,417 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:35496, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-07-22T12:34:45,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:34:45,422 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:43192, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-07-22T12:34:45,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:34:45,434 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:34:45,434 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-07-22T12:34:45,434 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-07-22T12:34:45,435 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(233): Backup: backup_1690029271161 hdfs://localhost:34745/backupUT/backup_1690029271161/default/test-1690029065625/ 2023-07-22T12:34:45,435 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-07-22T12:34:45,435 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:34:45,436 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster$12(2663): Client=jenkins//172.31.12.81 disable table1 2023-07-22T12:34:45,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=164, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=table1 2023-07-22T12:34:45,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=164 2023-07-22T12:34:45,440 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029285440"}]},"ts":"1690029285440"} 2023-07-22T12:34:45,441 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=DISABLING in hbase:meta 2023-07-22T12:34:45,442 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(293): Set table1 to state=DISABLING 2023-07-22T12:34:45,442 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=table1, region=de587539e56a809016723f0ce4633348, UNASSIGN}] 2023-07-22T12:34:45,444 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=165, ppid=164, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=table1, region=de587539e56a809016723f0ce4633348, UNASSIGN 2023-07-22T12:34:45,445 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=165 updating hbase:meta row=de587539e56a809016723f0ce4633348, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:45,445 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=166, ppid=165, state=RUNNABLE; CloseRegionProcedure de587539e56a809016723f0ce4633348, server=jenkins-hbase3.apache.org,43639,1690029056327}] 2023-07-22T12:34:45,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=164 2023-07-22T12:34:45,597 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] handler.UnassignRegionHandler(119): Close de587539e56a809016723f0ce4633348 2023-07-22T12:34:45,598 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegion(1668): Closing de587539e56a809016723f0ce4633348, disabling compactions & flushes 2023-07-22T12:34:45,598 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegion(1690): Closing region table1,,1690029280695.de587539e56a809016723f0ce4633348. 2023-07-22T12:34:45,598 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegion(1711): Waiting without time limit for close lock on table1,,1690029280695.de587539e56a809016723f0ce4633348. 2023-07-22T12:34:45,598 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegion(1778): Acquired close lock on table1,,1690029280695.de587539e56a809016723f0ce4633348. after waiting 0 ms 2023-07-22T12:34:45,598 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegion(1788): Updates disabled for region table1,,1690029280695.de587539e56a809016723f0ce4633348. 2023-07-22T12:34:45,599 DEBUG [StoreFileCloser-de587539e56a809016723f0ce4633348-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 1342e9345f484cc8a1dd24c0d41f72c9_SeqId_4_ 2023-07-22T12:34:45,604 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/table1/de587539e56a809016723f0ce4633348/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2023-07-22T12:34:45,604 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:34:45,604 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegion(1909): Closed table1,,1690029280695.de587539e56a809016723f0ce4633348. 2023-07-22T12:34:45,604 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegion(1622): Region close journal for de587539e56a809016723f0ce4633348: 2023-07-22T12:34:45,606 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] handler.UnassignRegionHandler(163): Closed de587539e56a809016723f0ce4633348 2023-07-22T12:34:45,606 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=165 updating hbase:meta row=de587539e56a809016723f0ce4633348, regionState=CLOSED 2023-07-22T12:34:45,609 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=166, resume processing ppid=165 2023-07-22T12:34:45,609 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=166, ppid=165, state=SUCCESS; CloseRegionProcedure de587539e56a809016723f0ce4633348, server=jenkins-hbase3.apache.org,43639,1690029056327 in 163 msec 2023-07-22T12:34:45,610 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=165, resume processing ppid=164 2023-07-22T12:34:45,610 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=165, ppid=164, state=SUCCESS; TransitRegionStateProcedure table=table1, region=de587539e56a809016723f0ce4633348, UNASSIGN in 167 msec 2023-07-22T12:34:45,611 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029285611"}]},"ts":"1690029285611"} 2023-07-22T12:34:45,611 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=DISABLED in hbase:meta 2023-07-22T12:34:45,613 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(305): Set table1 to state=DISABLED 2023-07-22T12:34:45,614 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=164, state=SUCCESS; DisableTableProcedure table=table1 in 177 msec 2023-07-22T12:34:45,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=164 2023-07-22T12:34:45,742 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster$5(2447): Client=jenkins//172.31.12.81 delete table1 2023-07-22T12:34:45,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=167, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=table1 2023-07-22T12:34:45,744 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=167, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=table1 2023-07-22T12:34:45,745 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=167, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=table1 2023-07-22T12:34:45,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=167 2023-07-22T12:34:45,746 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/table1/de587539e56a809016723f0ce4633348 2023-07-22T12:34:45,748 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/table1/de587539e56a809016723f0ce4633348/f, FileablePath, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/table1/de587539e56a809016723f0ce4633348/recovered.edits] 2023-07-22T12:34:45,752 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/table1/de587539e56a809016723f0ce4633348/f/1342e9345f484cc8a1dd24c0d41f72c9_SeqId_4_ to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/archive/data/default/table1/de587539e56a809016723f0ce4633348/f/1342e9345f484cc8a1dd24c0d41f72c9_SeqId_4_ 2023-07-22T12:34:45,754 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/table1/de587539e56a809016723f0ce4633348/recovered.edits/6.seqid to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/archive/data/default/table1/de587539e56a809016723f0ce4633348/recovered.edits/6.seqid 2023-07-22T12:34:45,755 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/table1/de587539e56a809016723f0ce4633348 2023-07-22T12:34:45,755 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(293): Archived table1 regions 2023-07-22T12:34:45,757 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=167, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=table1 2023-07-22T12:34:45,759 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of table1 from hbase:meta 2023-07-22T12:34:45,761 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(387): Removing 'table1' descriptor. 2023-07-22T12:34:45,762 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=167, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=table1 2023-07-22T12:34:45,762 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(377): Removing 'table1' from region states. 2023-07-22T12:34:45,762 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"table1,,1690029280695.de587539e56a809016723f0ce4633348.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1690029285762"}]},"ts":"9223372036854775807"} 2023-07-22T12:34:45,764 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-07-22T12:34:45,764 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => de587539e56a809016723f0ce4633348, NAME => 'table1,,1690029280695.de587539e56a809016723f0ce4633348.', STARTKEY => '', ENDKEY => ''}] 2023-07-22T12:34:45,764 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(381): Marking 'table1' as deleted. 2023-07-22T12:34:45,764 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1690029285764"}]},"ts":"9223372036854775807"} 2023-07-22T12:34:45,767 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1678): Deleted table table1 state from META 2023-07-22T12:34:45,768 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(127): Finished pid=167, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=table1 2023-07-22T12:34:45,769 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=167, state=SUCCESS; DeleteTableProcedure table=table1 in 26 msec 2023-07-22T12:34:45,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=167 2023-07-22T12:34:45,862 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreSingleCommand Thread=604 (was 602) - Thread LEAK? -, OpenFileDescriptor=687 (was 686) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=76 (was 90), ProcessCount=169 (was 169), AvailableMemoryMB=16133 (was 16216) 2023-07-22T12:34:45,862 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(130): Thread=604 is superior to 500 2023-07-22T12:34:45,877 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreMultipleDNE Thread=604, OpenFileDescriptor=687, MaxFileDescriptor=60000, SystemLoadAverage=76, ProcessCount=169, AvailableMemoryMB=16132 2023-07-22T12:34:45,877 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(130): Thread=604 is superior to 500 2023-07-22T12:34:45,877 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] backup.TestFullRestore(302): test restore fails on multiple tables that do not exist 2023-07-22T12:34:45,883 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@416882af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:34:45,887 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:34:45,889 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:43196, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:34:45,890 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:34:45,892 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:35504, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:34:45,900 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-07-22T12:34:45,905 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:34:45,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029285905 (current time:1690029285905). 2023-07-22T12:34:45,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:34:45,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-07-22T12:34:45,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b7711ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:34:45,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:34:45,912 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:43212, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:34:45,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:34:45,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:34:45,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=168, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:34:45,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 168 2023-07-22T12:34:45,915 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:34:45,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=168 2023-07-22T12:34:45,916 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:34:45,917 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:34:46,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=168 2023-07-22T12:34:46,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=168 2023-07-22T12:34:46,322 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:34:46,323 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33}] 2023-07-22T12:34:46,324 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=169, ppid=168, state=RUNNABLE; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33 2023-07-22T12:34:46,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43639 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=169 2023-07-22T12:34:46,476 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. 2023-07-22T12:34:46,476 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.HRegion(2819): Flushing 697cc5ecf0962f5ab9f4126febf7ce33 2/2 column families, dataSize=1.14 KB heapSize=2.22 KB 2023-07-22T12:34:46,480 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/meta/d6249f08554f41eda9e8435d77e87a57 is 169, key is trslm:hdfs://localhost:34745/backupUT\x00test-1690029065625/meta:log-roll-map/1690029280236/Put/seqid=0 2023-07-22T12:34:46,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=168 2023-07-22T12:34:46,884 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=451 B at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/meta/d6249f08554f41eda9e8435d77e87a57 2023-07-22T12:34:46,889 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: d6249f08554f41eda9e8435d77e87a57 2023-07-22T12:34:46,892 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/session/8262923b4b5f440087811f90a1a46b44 is 452, key is session:backup_1690029271161/session:context/1690029280656/Put/seqid=0 2023-07-22T12:34:47,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=168 2023-07-22T12:34:47,296 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=721 B at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/session/8262923b4b5f440087811f90a1a46b44 2023-07-22T12:34:47,301 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 8262923b4b5f440087811f90a1a46b44 2023-07-22T12:34:47,302 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/meta/d6249f08554f41eda9e8435d77e87a57 as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/d6249f08554f41eda9e8435d77e87a57 2023-07-22T12:34:47,305 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/d6249f08554f41eda9e8435d77e87a57, entries=4, sequenceid=138, filesize=5.5 K 2023-07-22T12:34:47,306 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/session/8262923b4b5f440087811f90a1a46b44 as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/8262923b4b5f440087811f90a1a46b44 2023-07-22T12:34:47,310 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/8262923b4b5f440087811f90a1a46b44, entries=2, sequenceid=138, filesize=5.4 K 2023-07-22T12:34:47,310 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1172, heapSize ~2.19 KB/2240, currentSize=0 B/0 for 697cc5ecf0962f5ab9f4126febf7ce33 in 834ms, sequenceid=138, compaction requested=true 2023-07-22T12:34:47,310 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.HRegion(2520): Flush status journal for 697cc5ecf0962f5ab9f4126febf7ce33: 2023-07-22T12:34:47,310 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. for snapshot_backup_system completed. 2023-07-22T12:34:47,310 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33.' region-info for snapshot=snapshot_backup_system 2023-07-22T12:34:47,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:34:47,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/b580ac8bece04b07a21605e555bdc696, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/5ea0158e2d574f758c68ee9c20c21191, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/a6d2695c5574440ea495f58a7c66ed03, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/8f78c6cced064299a30d72a0b4901528, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/606f117b1e384a54bb6dba5339e90e8f, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/15bf4f9b1b65467c96713e751468dc6c, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/b30f291cd15a4d92ac4b31f477e93428, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/3e46ffe2abec4ceba3ab7424eff1a910, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/e0ee030fbe0a48558b7d348517bf0f3c, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/93cd26fa4bd94fc8b3a8b6de32bded79, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/d6830dd5c21a4377bee0840e832a1db8, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/d6249f08554f41eda9e8435d77e87a57] hfiles 2023-07-22T12:34:47,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (1/12): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/b580ac8bece04b07a21605e555bdc696 for snapshot=snapshot_backup_system 2023-07-22T12:34:47,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (2/12): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/5ea0158e2d574f758c68ee9c20c21191 for snapshot=snapshot_backup_system 2023-07-22T12:34:47,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (3/12): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/a6d2695c5574440ea495f58a7c66ed03 for snapshot=snapshot_backup_system 2023-07-22T12:34:47,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (4/12): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/8f78c6cced064299a30d72a0b4901528 for snapshot=snapshot_backup_system 2023-07-22T12:34:47,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (5/12): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/606f117b1e384a54bb6dba5339e90e8f for snapshot=snapshot_backup_system 2023-07-22T12:34:47,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (6/12): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/15bf4f9b1b65467c96713e751468dc6c for snapshot=snapshot_backup_system 2023-07-22T12:34:47,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (7/12): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/b30f291cd15a4d92ac4b31f477e93428 for snapshot=snapshot_backup_system 2023-07-22T12:34:47,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (8/12): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/3e46ffe2abec4ceba3ab7424eff1a910 for snapshot=snapshot_backup_system 2023-07-22T12:34:47,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (9/12): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/e0ee030fbe0a48558b7d348517bf0f3c for snapshot=snapshot_backup_system 2023-07-22T12:34:47,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (10/12): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/93cd26fa4bd94fc8b3a8b6de32bded79 for snapshot=snapshot_backup_system 2023-07-22T12:34:47,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (11/12): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/d6830dd5c21a4377bee0840e832a1db8 for snapshot=snapshot_backup_system 2023-07-22T12:34:47,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (12/12): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/d6249f08554f41eda9e8435d77e87a57 for snapshot=snapshot_backup_system 2023-07-22T12:34:47,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/956010b3eaa64b4fa82ddea4557fac2c, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/0704967b71ce4a5cb0af553e6e47addb, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/62b5d2f1465b492ab802e26f5bf32f6b, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/c2fdeacbf4184719a11904adfeaf1fd8, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/660347589bb3465a82d9c9138151d6a4, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/2b4c40a0167b43f584d34520d18e9390, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/1550986ca81f443894a286cada73692c, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/a5d57e73d32d4eda8e52490df82775fa, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/83933de825724e92a18f2bfe93b821b6, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/82e877bb04384fa3a3f2266a337937ce, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/20a6ddbb9c214d998c32b0bb0820f924, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/0438ad092ea24358b05211a8e4572521, hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/8262923b4b5f440087811f90a1a46b44] hfiles 2023-07-22T12:34:47,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (1/13): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/956010b3eaa64b4fa82ddea4557fac2c for snapshot=snapshot_backup_system 2023-07-22T12:34:47,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (2/13): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/0704967b71ce4a5cb0af553e6e47addb for snapshot=snapshot_backup_system 2023-07-22T12:34:47,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (3/13): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/62b5d2f1465b492ab802e26f5bf32f6b for snapshot=snapshot_backup_system 2023-07-22T12:34:47,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (4/13): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/c2fdeacbf4184719a11904adfeaf1fd8 for snapshot=snapshot_backup_system 2023-07-22T12:34:47,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (5/13): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/660347589bb3465a82d9c9138151d6a4 for snapshot=snapshot_backup_system 2023-07-22T12:34:47,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (6/13): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/2b4c40a0167b43f584d34520d18e9390 for snapshot=snapshot_backup_system 2023-07-22T12:34:47,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (7/13): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/1550986ca81f443894a286cada73692c for snapshot=snapshot_backup_system 2023-07-22T12:34:47,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (8/13): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/a5d57e73d32d4eda8e52490df82775fa for snapshot=snapshot_backup_system 2023-07-22T12:34:47,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (9/13): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/83933de825724e92a18f2bfe93b821b6 for snapshot=snapshot_backup_system 2023-07-22T12:34:47,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (10/13): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/82e877bb04384fa3a3f2266a337937ce for snapshot=snapshot_backup_system 2023-07-22T12:34:47,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (11/13): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/20a6ddbb9c214d998c32b0bb0820f924 for snapshot=snapshot_backup_system 2023-07-22T12:34:47,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (12/13): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/0438ad092ea24358b05211a8e4572521 for snapshot=snapshot_backup_system 2023-07-22T12:34:47,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (13/13): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/8262923b4b5f440087811f90a1a46b44 for snapshot=snapshot_backup_system 2023-07-22T12:34:47,371 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-07-22T12:34:47,716 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. 2023-07-22T12:34:47,716 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=169 2023-07-22T12:34:47,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster(3994): Remote procedure done, pid=169 2023-07-22T12:34:47,716 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 697cc5ecf0962f5ab9f4126febf7ce33 2023-07-22T12:34:47,718 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=169, ppid=168, state=RUNNABLE; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33 2023-07-22T12:34:47,719 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=169, resume processing ppid=168 2023-07-22T12:34:47,719 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:34:47,719 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=169, ppid=168, state=SUCCESS; SnapshotRegionProcedure 697cc5ecf0962f5ab9f4126febf7ce33 in 1.3950 sec 2023-07-22T12:34:47,720 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:34:47,720 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:34:47,720 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-07-22T12:34:47,721 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_backup_system 2023-07-22T12:34:48,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=168 2023-07-22T12:34:48,135 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:34:48,149 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:34:48,149 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_backup_system 2023-07-22T12:34:48,966 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:34:48,966 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 168 2023-07-22T12:34:48,967 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=168, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 3.0530 sec 2023-07-22T12:34:50,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=168 2023-07-22T12:34:50,020 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(115): Backup backup_1690029285883 started at 1690029290020. 2023-07-22T12:34:50,022 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(119): Backup session backup_1690029285883 has been started. 2023-07-22T12:34:50,026 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-07-22T12:34:50,026 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-07-22T12:34:50,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-07-22T12:34:50,027 INFO [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-07-22T12:34:50,027 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-07-22T12:34:50,028 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-07-22T12:34:50,028 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-07-22T12:34:50,028 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:34:50,029 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:50,029 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:34:50,029 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:34:50,029 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-07-22T12:34:50,029 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-07-22T12:34:50,030 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-07-22T12:34:50,030 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:34:50,030 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-07-22T12:34:50,030 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-07-22T12:34:50,030 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-07-22T12:34:50,030 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-07-22T12:34:50,030 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-07-22T12:34:50,031 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-07-22T12:34:50,031 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-07-22T12:34:50,031 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,43639,1690029056327' joining acquired barrier for procedure (rolllog) in zk 2023-07-22T12:34:50,033 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-07-22T12:34:50,033 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:50,033 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:34:50,033 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:34:50,033 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-07-22T12:34:50,033 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:34:50,033 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:34:50,034 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:34:50,034 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:50,034 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:34:50,034 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,43639,1690029056327' joining acquired barrier for procedure 'rolllog' on coordinator 2023-07-22T12:34:50,034 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@6c71a547[Count = 0] remaining members to acquire global barrier 2023-07-22T12:34:50,034 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-07-22T12:34:50,034 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-07-22T12:34:50,035 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-07-22T12:34:50,035 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-07-22T12:34:50,036 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-07-22T12:34:50,036 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-07-22T12:34:50,036 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-07-22T12:34:50,036 DEBUG [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:50,036 INFO [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1690029275338 highest: 1690029275338 on jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:50,036 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta:.meta(num 1690029275315) roll requested 2023-07-22T12:34:50,055 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029290036.meta, exclude list is [], retry=0 2023-07-22T12:34:50,058 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK] 2023-07-22T12:34:50,059 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029275315.meta with entries=12, filesize=2.74 KB; new WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029290036.meta 2023-07-22T12:34:50,060 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK]] 2023-07-22T12:34:50,060 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta.1690029275315.meta is not closed yet, will try archiving it next time 2023-07-22T12:34:50,060 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C43639%2C1690029056327:(num 1690029275338) roll requested 2023-07-22T12:34:50,077 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029290060, exclude list is [], retry=0 2023-07-22T12:34:50,080 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK] 2023-07-22T12:34:50,107 DEBUG [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 71 2023-07-22T12:34:50,108 INFO [rs(jenkins-hbase3.apache.org,43639,1690029056327)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1690029290060 on jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:50,109 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029275338 with entries=13, filesize=3.13 KB; new WAL /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029290060 2023-07-22T12:34:50,109 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:39117,DS-711135b1-7150-45cc-b6bd-e4af6531d48e,DISK]] 2023-07-22T12:34:50,109 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029275338 is not closed yet, will try archiving it next time 2023-07-22T12:34:50,109 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/WALs/jenkins-hbase3.apache.org,43639,1690029056327/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029263844 to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/oldWALs/jenkins-hbase3.apache.org%2C43639%2C1690029056327.1690029263844 2023-07-22T12:34:50,115 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-07-22T12:34:50,115 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,43639,1690029056327' in zk 2023-07-22T12:34:50,119 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-07-22T12:34:50,119 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:34:50,119 DEBUG [member: 'jenkins-hbase3.apache.org,43639,1690029056327' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-07-22T12:34:50,119 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:50,120 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:34:50,120 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:34:50,120 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:34:50,120 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:34:50,121 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:34:50,121 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:50,121 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:34:50,121 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:34:50,122 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:50,122 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,43639,1690029056327': 2023-07-22T12:34:50,122 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,43639,1690029056327' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-07-22T12:34:50,122 INFO [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-07-22T12:34:50,122 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-07-22T12:34:50,122 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-07-22T12:34:50,122 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-07-22T12:34:50,122 INFO [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-07-22T12:34:50,124 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-07-22T12:34:50,124 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-07-22T12:34:50,124 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:34:50,124 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:34:50,124 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:34:50,124 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:34:50,124 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:34:50,124 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:34:50,124 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:34:50,124 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:34:50,125 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:34:50,125 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:34:50,126 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:50,126 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:34:50,126 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:34:50,126 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:34:50,130 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:34:50,130 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:34:50,131 DEBUG [(jenkins-hbase3.apache.org,37923,1690029053050)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:34:50,131 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-07-22T12:34:50,131 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-07-22T12:34:50,131 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:34:50,131 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:34:50,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-07-22T12:34:50,235 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1690029290235_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } 2023-07-22T12:34:50,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029290235 (current time:1690029290235). 2023-07-22T12:34:50,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:34:50,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1690029290235_ns2_test-16900290656251 VERSION not specified, setting to 2 2023-07-22T12:34:50,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59148aea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:34:50,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:34:50,249 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:45310, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:34:50,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:34:50,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:34:50,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=170, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1690029290235_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } 2023-07-22T12:34:50,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1690029290235_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 }, snapshot procedure id = 170 2023-07-22T12:34:50,253 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1690029290235_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:34:50,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=170 2023-07-22T12:34:50,254 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1690029290235_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:34:50,255 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1690029290235_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:34:50,265 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1690029290235_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:34:50,265 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE; SnapshotRegionProcedure 01e992ceb654b1e583b990da186e85a8}] 2023-07-22T12:34:50,267 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=171, ppid=170, state=RUNNABLE; SnapshotRegionProcedure 01e992ceb654b1e583b990da186e85a8 2023-07-22T12:34:50,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=170 2023-07-22T12:34:50,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43639 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=171 2023-07-22T12:34:50,418 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-16900290656251,,1690029231061.01e992ceb654b1e583b990da186e85a8. 2023-07-22T12:34:50,418 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] regionserver.HRegion(2520): Flush status journal for 01e992ceb654b1e583b990da186e85a8: 2023-07-22T12:34:50,418 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-16900290656251,,1690029231061.01e992ceb654b1e583b990da186e85a8. for snapshot_1690029290235_ns2_test-16900290656251 completed. 2023-07-22T12:34:50,419 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] snapshot.SnapshotManifest(240): Storing 'ns2:test-16900290656251,,1690029231061.01e992ceb654b1e583b990da186e85a8.' region-info for snapshot=snapshot_1690029290235_ns2_test-16900290656251 2023-07-22T12:34:50,419 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:34:50,419 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/01e992ceb654b1e583b990da186e85a8/f/d63801d92d8645ef95d1060b6ba3bbd9_SeqId_4_] hfiles 2023-07-22T12:34:50,419 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/01e992ceb654b1e583b990da186e85a8/f/d63801d92d8645ef95d1060b6ba3bbd9_SeqId_4_ for snapshot=snapshot_1690029290235_ns2_test-16900290656251 2023-07-22T12:34:50,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=170 2023-07-22T12:34:50,826 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-16900290656251,,1690029231061.01e992ceb654b1e583b990da186e85a8. 2023-07-22T12:34:50,826 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=171 2023-07-22T12:34:50,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster(3994): Remote procedure done, pid=171 2023-07-22T12:34:50,826 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1690029290235_ns2_test-16900290656251 on region 01e992ceb654b1e583b990da186e85a8 2023-07-22T12:34:50,827 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=171, ppid=170, state=RUNNABLE; SnapshotRegionProcedure 01e992ceb654b1e583b990da186e85a8 2023-07-22T12:34:50,829 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=171, resume processing ppid=170 2023-07-22T12:34:50,829 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=171, ppid=170, state=SUCCESS; SnapshotRegionProcedure 01e992ceb654b1e583b990da186e85a8 in 563 msec 2023-07-22T12:34:50,829 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1690029290235_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:34:50,830 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1690029290235_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:34:50,830 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1690029290235_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:34:50,830 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1690029290235_ns2_test-16900290656251 2023-07-22T12:34:50,831 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_1690029290235_ns2_test-16900290656251 2023-07-22T12:34:50,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=170 2023-07-22T12:34:51,239 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1690029290235_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:34:51,243 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1690029290235_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:34:51,243 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_1690029290235_ns2_test-16900290656251 to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029290235_ns2_test-16900290656251 2023-07-22T12:34:51,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=170 2023-07-22T12:34:52,058 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1690029290235_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:34:52,058 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1690029290235_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 }, snapshot procedure id = 170 2023-07-22T12:34:52,059 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=170, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1690029290235_ns2_test-16900290656251 table=ns2:test-16900290656251 type=FLUSH ttl=0 } in 1.8070 sec 2023-07-22T12:34:52,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=170 2023-07-22T12:34:52,359 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1690029292359_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } 2023-07-22T12:34:52,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029292359 (current time:1690029292359). 2023-07-22T12:34:52,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:34:52,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1690029292359_ns3_test-16900290656252 VERSION not specified, setting to 2 2023-07-22T12:34:52,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c3ae88e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:34:52,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:34:52,367 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:45326, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:34:52,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:34:52,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:34:52,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure2.ProcedureExecutor(1033): Stored pid=172, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1690029292359_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } 2023-07-22T12:34:52,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1690029292359_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 }, snapshot procedure id = 172 2023-07-22T12:34:52,370 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1690029292359_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:34:52,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=172 2023-07-22T12:34:52,371 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1690029292359_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:34:52,372 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1690029292359_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:34:52,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=172 2023-07-22T12:34:52,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=172 2023-07-22T12:34:52,777 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1690029292359_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:34:52,777 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE; SnapshotRegionProcedure 7e12e84cd2aa3dccee1c81fd5a1ead69}] 2023-07-22T12:34:52,779 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=173, ppid=172, state=RUNNABLE; SnapshotRegionProcedure 7e12e84cd2aa3dccee1c81fd5a1ead69 2023-07-22T12:34:52,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43639 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=173 2023-07-22T12:34:52,930 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=173}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-16900290656252,,1690029235139.7e12e84cd2aa3dccee1c81fd5a1ead69. 2023-07-22T12:34:52,931 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=173}] regionserver.HRegion(2520): Flush status journal for 7e12e84cd2aa3dccee1c81fd5a1ead69: 2023-07-22T12:34:52,931 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=173}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-16900290656252,,1690029235139.7e12e84cd2aa3dccee1c81fd5a1ead69. for snapshot_1690029292359_ns3_test-16900290656252 completed. 2023-07-22T12:34:52,931 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=173}] snapshot.SnapshotManifest(240): Storing 'ns3:test-16900290656252,,1690029235139.7e12e84cd2aa3dccee1c81fd5a1ead69.' region-info for snapshot=snapshot_1690029292359_ns3_test-16900290656252 2023-07-22T12:34:52,931 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=173}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:34:52,931 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=173}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-07-22T12:34:52,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=172 2023-07-22T12:34:53,225 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_table1 2023-07-22T12:34:53,336 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=173}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-16900290656252,,1690029235139.7e12e84cd2aa3dccee1c81fd5a1ead69. 2023-07-22T12:34:53,336 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=173 2023-07-22T12:34:53,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.HMaster(3994): Remote procedure done, pid=173 2023-07-22T12:34:53,337 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1690029292359_ns3_test-16900290656252 on region 7e12e84cd2aa3dccee1c81fd5a1ead69 2023-07-22T12:34:53,338 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=173, ppid=172, state=RUNNABLE; SnapshotRegionProcedure 7e12e84cd2aa3dccee1c81fd5a1ead69 2023-07-22T12:34:53,340 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=173, resume processing ppid=172 2023-07-22T12:34:53,340 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1690029292359_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:34:53,340 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=173, ppid=172, state=SUCCESS; SnapshotRegionProcedure 7e12e84cd2aa3dccee1c81fd5a1ead69 in 562 msec 2023-07-22T12:34:53,340 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1690029292359_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:34:53,341 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1690029292359_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:34:53,341 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1690029292359_ns3_test-16900290656252 2023-07-22T12:34:53,341 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_1690029292359_ns3_test-16900290656252 2023-07-22T12:34:53,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=172 2023-07-22T12:34:53,749 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1690029292359_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:34:53,753 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1690029292359_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:34:53,753 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/.tmp/snapshot_1690029292359_ns3_test-16900290656252 to hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029292359_ns3_test-16900290656252 2023-07-22T12:34:54,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=172 2023-07-22T12:34:54,569 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1690029292359_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:34:54,569 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1690029292359_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 }, snapshot procedure id = 172 2023-07-22T12:34:54,570 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=172, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1690029292359_ns3_test-16900290656252 table=ns3:test-16900290656252 type=FLUSH ttl=0 } in 2.2010 sec 2023-07-22T12:34:56,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=172 2023-07-22T12:34:56,476 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1690029285883 2023-07-22T12:34:56,476 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-07-22T12:34:56,476 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2023-07-22T12:34:56,476 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1690029285883_ns2:test-16900290656251 2023-07-22T12:34:56,476 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1690029290235_ns2_test-16900290656251 to hdfs://localhost:34745/backupUT/backup_1690029285883/ns2/test-16900290656251/ 2023-07-22T12:34:56,476 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-07-22T12:34:56,500 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:34745, inputRoot=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947 2023-07-22T12:34:56,500 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,37923,1690029053050_-21732621_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:34745/backupUT/backup_1690029285883/ns2/test-16900290656251, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:34745/backupUT/backup_1690029285883/ns2/test-16900290656251/.hbase-snapshot/.tmp/snapshot_1690029290235_ns2_test-16900290656251 2023-07-22T12:34:56,500 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:34745, inputRoot=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947, snapshotDir=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029290235_ns2_test-16900290656251. 2023-07-22T12:34:56,506 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029290235_ns2_test-16900290656251 to hdfs://localhost:34745/backupUT/backup_1690029285883/ns2/test-16900290656251/.hbase-snapshot/.tmp/snapshot_1690029290235_ns2_test-16900290656251 2023-07-22T12:34:56,921 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:56,921 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:56,922 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:56,922 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:57,896 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/hadoop-6155651273132665825.jar 2023-07-22T12:34:57,897 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:57,897 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:57,897 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:57,898 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:57,898 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:57,898 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:57,899 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:57,899 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:57,899 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:57,900 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:34:57,900 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-07-22T12:34:57,901 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-07-22T12:34:57,901 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-07-22T12:34:57,901 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-07-22T12:34:57,902 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-07-22T12:34:57,902 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-07-22T12:34:57,902 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-07-22T12:34:57,903 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-07-22T12:34:57,903 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-07-22T12:34:57,904 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-07-22T12:34:57,904 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-07-22T12:34:57,904 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:34:57,905 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:34:57,905 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:34:57,906 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:34:57,906 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:34:57,906 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:34:57,907 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:34:57,949 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1690029290235_ns2_test-16900290656251' hfile list 2023-07-22T12:34:57,950 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-07-22T12:34:58,021 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029297986/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-07-22T12:34:58,021 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029297986/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-07-22T12:34:58,074 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-07-22T12:34:58,100 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/01e992ceb654b1e583b990da186e85a8/f/d63801d92d8645ef95d1060b6ba3bbd9_SeqId_4_ output=hdfs://localhost:34745/backupUT/backup_1690029285883/ns2/test-16900290656251/archive/data/ns2/test-16900290656251/01e992ceb654b1e583b990da186e85a8/f/d63801d92d8645ef95d1060b6ba3bbd9_SeqId_4_ 2023-07-22T12:34:58,100 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8724 (8.5 K) time=0sec 8.320M/sec 2023-07-22T12:34:58,740 DEBUG [regionserver/jenkins-hbase3:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.7692307692307693, tune throughput to 88.46 MB/second 2023-07-22T12:34:58,763 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-07-22T12:34:59,058 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-07-22T12:34:59,058 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-07-22T12:34:59,063 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1690029290235_ns2_test-16900290656251 2023-07-22T12:34:59,064 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1690029290235_ns2_test-16900290656251 finished. 2023-07-22T12:34:59,064 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1690029285883_ns3:test-16900290656252 2023-07-22T12:34:59,064 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1690029292359_ns3_test-16900290656252 to hdfs://localhost:34745/backupUT/backup_1690029285883/ns3/test-16900290656252/ 2023-07-22T12:34:59,064 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-07-22T12:34:59,088 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:34745, inputRoot=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947 2023-07-22T12:34:59,088 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,37923,1690029053050_-1386541424_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:34745/backupUT/backup_1690029285883/ns3/test-16900290656252, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:34745/backupUT/backup_1690029285883/ns3/test-16900290656252/.hbase-snapshot/.tmp/snapshot_1690029292359_ns3_test-16900290656252 2023-07-22T12:34:59,088 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:34745, inputRoot=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947, snapshotDir=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029292359_ns3_test-16900290656252. 2023-07-22T12:34:59,093 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/.hbase-snapshot/snapshot_1690029292359_ns3_test-16900290656252 to hdfs://localhost:34745/backupUT/backup_1690029285883/ns3/test-16900290656252/.hbase-snapshot/.tmp/snapshot_1690029292359_ns3_test-16900290656252 2023-07-22T12:34:59,129 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:59,130 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:59,131 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:59,131 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:00,130 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/hadoop-689395805593486959.jar 2023-07-22T12:35:00,131 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:00,131 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:00,131 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:00,132 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:00,132 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:00,132 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:00,133 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:00,133 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:00,133 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:00,134 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:35:00,135 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-07-22T12:35:00,135 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-07-22T12:35:00,135 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-07-22T12:35:00,136 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-07-22T12:35:00,136 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-07-22T12:35:00,136 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-07-22T12:35:00,136 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-07-22T12:35:00,137 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-07-22T12:35:00,137 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-07-22T12:35:00,138 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-07-22T12:35:00,138 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-07-22T12:35:00,138 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:35:00,139 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:35:00,140 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:35:00,140 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:35:00,140 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:35:00,141 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:35:00,141 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:35:00,182 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1690029292359_ns3_test-16900290656252' hfile list 2023-07-22T12:35:00,250 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029300218/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-07-22T12:35:00,250 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/mapreduce.cluster.local.dir/1690029300218/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-07-22T12:35:01,278 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-07-22T12:35:01,279 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-07-22T12:35:01,284 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1690029292359_ns3_test-16900290656252 2023-07-22T12:35:01,284 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1690029292359_ns3_test-16900290656252 finished. 2023-07-22T12:35:01,284 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(1014): ns2:test-16900290656251 2023-07-22T12:35:01,284 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(1014): ns3:test-16900290656252 2023-07-22T12:35:01,290 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029285883 2023-07-22T12:35:01,290 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-07-22T12:35:01,694 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:34745/backupUT/backup_1690029285883/.backup.manifest 2023-07-22T12:35:01,694 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029285883 2023-07-22T12:35:01,694 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-07-22T12:35:02,099 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:34745/backupUT/backup_1690029285883/.backup.manifest 2023-07-22T12:35:02,099 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(358): Backup backup_1690029285883 finished: type=FULL,tablelist=ns2:test-16900290656251;ns3:test-16900290656252,targetRootDir=hdfs://localhost:34745/backupUT,startts=1690029290020,completets=1690029301289,bytescopied=0 2023-07-22T12:35:02,099 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-07-22T12:35:02,099 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1690029290235_ns2_test-16900290656251 2023-07-22T12:35:02,100 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1690029290235_ns2_test-16900290656251" 2023-07-22T12:35:02,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1690029290235_ns2_test-16900290656251 2023-07-22T12:35:02,103 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1690029290235_ns2_test-16900290656251 for backup backup_1690029285883 succeeded. 2023-07-22T12:35:02,103 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1690029292359_ns3_test-16900290656252 2023-07-22T12:35:02,103 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1690029292359_ns3_test-16900290656252" 2023-07-22T12:35:02,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1690029292359_ns3_test-16900290656252 2023-07-22T12:35:02,105 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1690029292359_ns3_test-16900290656252 for backup backup_1690029285883 succeeded. 2023-07-22T12:35:02,106 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-07-22T12:35:02,109 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-07-22T12:35:02,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37923 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-07-22T12:35:02,111 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-07-22T12:35:02,112 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-07-22T12:35:02,116 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.TableBackupClient(376): Backup backup_1690029285883 completed. 2023-07-22T12:35:02,117 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:35:02,125 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(101): HBase table ns2:table2 does not exist. It will be created during restore process 2023-07-22T12:35:02,126 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(101): HBase table ns3:table3_restore does not exist. It will be created during restore process 2023-07-22T12:35:02,126 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34745/backupUT/backup_1690029285883/.backup.manifest 2023-07-22T12:35:02,128 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029285883/.backup.manifest 2023-07-22T12:35:02,128 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34745/backupUT/backup_1690029285883/.backup.manifest 2023-07-22T12:35:02,130 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029285883/.backup.manifest 2023-07-22T12:35:02,130 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34745/backupUT/backup_1690029285883/.backup.manifest 2023-07-22T12:35:02,132 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029285883/.backup.manifest 2023-07-22T12:35:02,132 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] impl.RestoreTablesClient(148): Restoring 'faketable1' to 'ns2:table2' from full backup image hdfs://localhost:34745/backupUT/backup_1690029285883/default/faketable1 2023-07-22T12:35:02,149 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreMultipleDNE Thread=603 (was 604), OpenFileDescriptor=694 (was 687) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=74 (was 76), ProcessCount=169 (was 169), AvailableMemoryMB=16095 (was 16132) 2023-07-22T12:35:02,149 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.ResourceChecker(130): Thread=603 is superior to 500 2023-07-22T12:35:02,156 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2023-07-22T12:35:02,156 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:35:02,156 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2023-07-22T12:35:02,156 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] util.JVMClusterUtil(257): Found active master hash=1501223713, stopped=false 2023-07-22T12:35:02,156 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] master.ServerManager(910): Cluster shutdown requested of master=jenkins-hbase3.apache.org,37923,1690029053050 2023-07-22T12:35:02,160 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] procedure2.ProcedureExecutor(631): Stopping 2023-07-22T12:35:02,160 DEBUG [Listener at jenkins-hbase3.apache.org/43645 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:35:02,160 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] regionserver.HRegionServer(2513): ***** STOPPING region server 'jenkins-hbase3.apache.org,43639,1690029056327' ***** 2023-07-22T12:35:02,161 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] regionserver.HRegionServer(2527): STOPPED: Shutdown requested 2023-07-22T12:35:02,161 INFO [RS:0;jenkins-hbase3:43639 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-07-22T12:35:02,174 INFO [RS:0;jenkins-hbase3:43639 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@59d42308{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-07-22T12:35:02,179 INFO [RS:0;jenkins-hbase3:43639 {}] server.AbstractConnector(383): Stopped ServerConnector@82c78f1{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-07-22T12:35:02,179 INFO [RS:0;jenkins-hbase3:43639 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-07-22T12:35:02,180 INFO [RS:0;jenkins-hbase3:43639 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@3cbef607{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-07-22T12:35:02,181 INFO [RS:0;jenkins-hbase3:43639 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@727639c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/hadoop.log.dir/,STOPPED} 2023-07-22T12:35:02,183 INFO [RS:0;jenkins-hbase3:43639 {}] regionserver.HeapMemoryManager(220): Stopping 2023-07-22T12:35:02,183 INFO [RS:0;jenkins-hbase3:43639 {}] flush.RegionServerFlushTableProcedureManager(117): Stopping region server flush procedure manager gracefully. 2023-07-22T12:35:02,184 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2023-07-22T12:35:02,184 INFO [RS:0;jenkins-hbase3:43639 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2023-07-22T12:35:02,184 INFO [RS:0;jenkins-hbase3:43639 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2023-07-22T12:35:02,184 INFO [RS:0;jenkins-hbase3:43639 {}] regionserver.HRegionServer(3524): Received CLOSE for 54af1049fa09c4790d7061c581618a59 2023-07-22T12:35:02,184 INFO [RS:0;jenkins-hbase3:43639 {}] regionserver.HRegionServer(3524): Received CLOSE for 697cc5ecf0962f5ab9f4126febf7ce33 2023-07-22T12:35:02,184 INFO [RS:0;jenkins-hbase3:43639 {}] regionserver.HRegionServer(3524): Received CLOSE for fc6cb6b064d3fb792c9e01da5e382760 2023-07-22T12:35:02,184 INFO [RS:0;jenkins-hbase3:43639 {}] regionserver.HRegionServer(3524): Received CLOSE for 7e12e84cd2aa3dccee1c81fd5a1ead69 2023-07-22T12:35:02,184 INFO [RS:0;jenkins-hbase3:43639 {}] regionserver.HRegionServer(3524): Received CLOSE for 595143905c5328f814fba73ad7017c76 2023-07-22T12:35:02,185 INFO [RS:0;jenkins-hbase3:43639 {}] regionserver.HRegionServer(3524): Received CLOSE for baf4f0d945c66abea73108272203bc6e 2023-07-22T12:35:02,185 INFO [RS:0;jenkins-hbase3:43639 {}] regionserver.HRegionServer(3524): Received CLOSE for 01e992ceb654b1e583b990da186e85a8 2023-07-22T12:35:02,185 INFO [RS:0;jenkins-hbase3:43639 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,43639,1690029056327 2023-07-22T12:35:02,185 DEBUG [RS:0;jenkins-hbase3:43639 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:35:02,185 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 54af1049fa09c4790d7061c581618a59, disabling compactions & flushes 2023-07-22T12:35:02,185 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns4:test-16900290656253,,1690029072701.54af1049fa09c4790d7061c581618a59. 2023-07-22T12:35:02,185 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16900290656253,,1690029072701.54af1049fa09c4790d7061c581618a59. 2023-07-22T12:35:02,185 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16900290656253,,1690029072701.54af1049fa09c4790d7061c581618a59. after waiting 0 ms 2023-07-22T12:35:02,185 INFO [RS:0;jenkins-hbase3:43639 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2023-07-22T12:35:02,185 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16900290656253,,1690029072701.54af1049fa09c4790d7061c581618a59. 2023-07-22T12:35:02,185 INFO [RS:0;jenkins-hbase3:43639 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2023-07-22T12:35:02,185 INFO [RS:0;jenkins-hbase3:43639 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2023-07-22T12:35:02,186 INFO [RS:0;jenkins-hbase3:43639 {}] regionserver.HRegionServer(3524): Received CLOSE for 1588230740 2023-07-22T12:35:02,186 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2023-07-22T12:35:02,186 INFO [RS:0;jenkins-hbase3:43639 {}] regionserver.HRegionServer(1579): Waiting on 8 regions to close 2023-07-22T12:35:02,186 DEBUG [RS:0;jenkins-hbase3:43639 {}] regionserver.HRegionServer(1583): Online Regions={54af1049fa09c4790d7061c581618a59=ns4:test-16900290656253,,1690029072701.54af1049fa09c4790d7061c581618a59., 697cc5ecf0962f5ab9f4126febf7ce33=backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33., fc6cb6b064d3fb792c9e01da5e382760=backup:system_bulk,,1690029077151.fc6cb6b064d3fb792c9e01da5e382760., 7e12e84cd2aa3dccee1c81fd5a1ead69=ns3:test-16900290656252,,1690029235139.7e12e84cd2aa3dccee1c81fd5a1ead69., 1588230740=hbase:meta,,1.1588230740, 595143905c5328f814fba73ad7017c76=hbase:namespace,,1690029059700.595143905c5328f814fba73ad7017c76., baf4f0d945c66abea73108272203bc6e=test-1690029065625,,1690029166757.baf4f0d945c66abea73108272203bc6e., 01e992ceb654b1e583b990da186e85a8=ns2:test-16900290656251,,1690029231061.01e992ceb654b1e583b990da186e85a8.} 2023-07-22T12:35:02,187 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-07-22T12:35:02,187 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-07-22T12:35:02,187 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-07-22T12:35:02,187 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-07-22T12:35:02,189 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-07-22T12:35:02,190 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2819): Flushing 1588230740 3/3 column families, dataSize=38.00 KB heapSize=66.30 KB 2023-07-22T12:35:02,190 DEBUG [RS:0;jenkins-hbase3:43639 {}] regionserver.HRegionServer(1609): Waiting on 01e992ceb654b1e583b990da186e85a8, 1588230740, 54af1049fa09c4790d7061c581618a59, 595143905c5328f814fba73ad7017c76, 697cc5ecf0962f5ab9f4126febf7ce33, 7e12e84cd2aa3dccee1c81fd5a1ead69, baf4f0d945c66abea73108272203bc6e, fc6cb6b064d3fb792c9e01da5e382760 2023-07-22T12:35:02,197 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns4/test-16900290656253/54af1049fa09c4790d7061c581618a59/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-07-22T12:35:02,198 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:35:02,199 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns4:test-16900290656253,,1690029072701.54af1049fa09c4790d7061c581618a59. 2023-07-22T12:35:02,199 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 54af1049fa09c4790d7061c581618a59: 2023-07-22T12:35:02,199 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns4:test-16900290656253,,1690029072701.54af1049fa09c4790d7061c581618a59. 2023-07-22T12:35:02,199 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 697cc5ecf0962f5ab9f4126febf7ce33, disabling compactions & flushes 2023-07-22T12:35:02,199 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. 2023-07-22T12:35:02,199 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. 2023-07-22T12:35:02,199 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. after waiting 0 ms 2023-07-22T12:35:02,199 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. 2023-07-22T12:35:02,200 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 697cc5ecf0962f5ab9f4126febf7ce33 2/2 column families, dataSize=1.46 KB heapSize=2.61 KB 2023-07-22T12:35:02,203 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/meta/b3a18477150e47e680b9d703952e9368 is 171, key is trslm:hdfs://localhost:34745/backupUT\x00ns2:test-16900290656251/meta:log-roll-map/1690029301286/Put/seqid=0 2023-07-22T12:35:02,215 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=717 B at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/meta/b3a18477150e47e680b9d703952e9368 2023-07-22T12:35:02,220 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: b3a18477150e47e680b9d703952e9368 2023-07-22T12:35:02,222 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/session/a7d8bd7134694f8190bbe86f0b8f23b8 is 524, key is session:backup_1690029285883/session:context/1690029302111/Put/seqid=0 2023-07-22T12:35:02,223 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/hbase/meta/1588230740/.tmp/info/32c7f6f276b9434e98e87e5bef319158 is 159, key is ns2:test-16900290656251,,1690029231061.01e992ceb654b1e583b990da186e85a8./info:regioninfo/1690029231810/Put/seqid=0 2023-07-22T12:35:02,390 DEBUG [RS:0;jenkins-hbase3:43639 {}] regionserver.HRegionServer(1609): Waiting on 01e992ceb654b1e583b990da186e85a8, 1588230740, 595143905c5328f814fba73ad7017c76, 697cc5ecf0962f5ab9f4126febf7ce33, 7e12e84cd2aa3dccee1c81fd5a1ead69, baf4f0d945c66abea73108272203bc6e, fc6cb6b064d3fb792c9e01da5e382760 2023-07-22T12:35:02,591 DEBUG [RS:0;jenkins-hbase3:43639 {}] regionserver.HRegionServer(1609): Waiting on 01e992ceb654b1e583b990da186e85a8, 1588230740, 595143905c5328f814fba73ad7017c76, 697cc5ecf0962f5ab9f4126febf7ce33, 7e12e84cd2aa3dccee1c81fd5a1ead69, baf4f0d945c66abea73108272203bc6e, fc6cb6b064d3fb792c9e01da5e382760 2023-07-22T12:35:02,633 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=775 B at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/session/a7d8bd7134694f8190bbe86f0b8f23b8 2023-07-22T12:35:02,634 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.30 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/hbase/meta/1588230740/.tmp/info/32c7f6f276b9434e98e87e5bef319158 2023-07-22T12:35:02,638 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a7d8bd7134694f8190bbe86f0b8f23b8 2023-07-22T12:35:02,638 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 32c7f6f276b9434e98e87e5bef319158 2023-07-22T12:35:02,639 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/meta/b3a18477150e47e680b9d703952e9368 as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/b3a18477150e47e680b9d703952e9368 2023-07-22T12:35:02,644 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/meta/b3a18477150e47e680b9d703952e9368, entries=6, sequenceid=148, filesize=5.8 K 2023-07-22T12:35:02,645 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/.tmp/session/a7d8bd7134694f8190bbe86f0b8f23b8 as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/a7d8bd7134694f8190bbe86f0b8f23b8 2023-07-22T12:35:02,650 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/session/a7d8bd7134694f8190bbe86f0b8f23b8, entries=2, sequenceid=148, filesize=5.5 K 2023-07-22T12:35:02,651 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~1.46 KB/1492, heapSize ~2.58 KB/2640, currentSize=0 B/0 for 697cc5ecf0962f5ab9f4126febf7ce33 in 452ms, sequenceid=148, compaction requested=true 2023-07-22T12:35:02,652 DEBUG [StoreFileCloser-697cc5ecf0962f5ab9f4126febf7ce33-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: b580ac8bece04b07a21605e555bdc696 2023-07-22T12:35:02,654 DEBUG [StoreFileCloser-697cc5ecf0962f5ab9f4126febf7ce33-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 5ea0158e2d574f758c68ee9c20c21191 2023-07-22T12:35:02,656 DEBUG [StoreFileCloser-697cc5ecf0962f5ab9f4126febf7ce33-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a6d2695c5574440ea495f58a7c66ed03 2023-07-22T12:35:02,658 DEBUG [StoreFileCloser-697cc5ecf0962f5ab9f4126febf7ce33-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 8f78c6cced064299a30d72a0b4901528 2023-07-22T12:35:02,659 DEBUG [StoreFileCloser-697cc5ecf0962f5ab9f4126febf7ce33-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 606f117b1e384a54bb6dba5339e90e8f 2023-07-22T12:35:02,661 DEBUG [StoreFileCloser-697cc5ecf0962f5ab9f4126febf7ce33-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 15bf4f9b1b65467c96713e751468dc6c 2023-07-22T12:35:02,662 DEBUG [StoreFileCloser-697cc5ecf0962f5ab9f4126febf7ce33-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: b30f291cd15a4d92ac4b31f477e93428 2023-07-22T12:35:02,664 DEBUG [StoreFileCloser-697cc5ecf0962f5ab9f4126febf7ce33-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 3e46ffe2abec4ceba3ab7424eff1a910 2023-07-22T12:35:02,665 DEBUG [StoreFileCloser-697cc5ecf0962f5ab9f4126febf7ce33-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: e0ee030fbe0a48558b7d348517bf0f3c 2023-07-22T12:35:02,667 DEBUG [StoreFileCloser-697cc5ecf0962f5ab9f4126febf7ce33-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 93cd26fa4bd94fc8b3a8b6de32bded79 2023-07-22T12:35:02,668 DEBUG [StoreFileCloser-697cc5ecf0962f5ab9f4126febf7ce33-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: d6830dd5c21a4377bee0840e832a1db8 2023-07-22T12:35:02,671 DEBUG [StoreFileCloser-697cc5ecf0962f5ab9f4126febf7ce33-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: d6249f08554f41eda9e8435d77e87a57 2023-07-22T12:35:02,675 DEBUG [StoreFileCloser-697cc5ecf0962f5ab9f4126febf7ce33-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: b3a18477150e47e680b9d703952e9368 2023-07-22T12:35:02,675 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/hbase/meta/1588230740/.tmp/rep_barrier/e06b637a18df4b08a2ac3e4952bee84f is 107, key is ns2:test-16900290656251,,1690029068374.0112d97ae6696b92341dfd7ff8824a0a./rep_barrier:/1690029096060/DeleteFamily/seqid=0 2023-07-22T12:35:02,675 DEBUG [StoreFileCloser-697cc5ecf0962f5ab9f4126febf7ce33-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 956010b3eaa64b4fa82ddea4557fac2c 2023-07-22T12:35:02,676 DEBUG [StoreFileCloser-697cc5ecf0962f5ab9f4126febf7ce33-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 0704967b71ce4a5cb0af553e6e47addb 2023-07-22T12:35:02,676 DEBUG [StoreFileCloser-697cc5ecf0962f5ab9f4126febf7ce33-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 62b5d2f1465b492ab802e26f5bf32f6b 2023-07-22T12:35:02,676 DEBUG [StoreFileCloser-697cc5ecf0962f5ab9f4126febf7ce33-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: c2fdeacbf4184719a11904adfeaf1fd8 2023-07-22T12:35:02,676 DEBUG [StoreFileCloser-697cc5ecf0962f5ab9f4126febf7ce33-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 660347589bb3465a82d9c9138151d6a4 2023-07-22T12:35:02,676 DEBUG [StoreFileCloser-697cc5ecf0962f5ab9f4126febf7ce33-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 2b4c40a0167b43f584d34520d18e9390 2023-07-22T12:35:02,676 DEBUG [StoreFileCloser-697cc5ecf0962f5ab9f4126febf7ce33-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 1550986ca81f443894a286cada73692c 2023-07-22T12:35:02,676 DEBUG [StoreFileCloser-697cc5ecf0962f5ab9f4126febf7ce33-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a5d57e73d32d4eda8e52490df82775fa 2023-07-22T12:35:02,676 DEBUG [StoreFileCloser-697cc5ecf0962f5ab9f4126febf7ce33-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 83933de825724e92a18f2bfe93b821b6 2023-07-22T12:35:02,676 DEBUG [StoreFileCloser-697cc5ecf0962f5ab9f4126febf7ce33-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 82e877bb04384fa3a3f2266a337937ce 2023-07-22T12:35:02,676 DEBUG [StoreFileCloser-697cc5ecf0962f5ab9f4126febf7ce33-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 20a6ddbb9c214d998c32b0bb0820f924 2023-07-22T12:35:02,677 DEBUG [StoreFileCloser-697cc5ecf0962f5ab9f4126febf7ce33-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 0438ad092ea24358b05211a8e4572521 2023-07-22T12:35:02,677 DEBUG [StoreFileCloser-697cc5ecf0962f5ab9f4126febf7ce33-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 8262923b4b5f440087811f90a1a46b44 2023-07-22T12:35:02,677 DEBUG [StoreFileCloser-697cc5ecf0962f5ab9f4126febf7ce33-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a7d8bd7134694f8190bbe86f0b8f23b8 2023-07-22T12:35:02,680 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system/697cc5ecf0962f5ab9f4126febf7ce33/recovered.edits/151.seqid, newMaxSeqId=151, maxSeqId=1 2023-07-22T12:35:02,681 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:35:02,682 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. 2023-07-22T12:35:02,682 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 697cc5ecf0962f5ab9f4126febf7ce33: 2023-07-22T12:35:02,682 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system,,1690029075023.697cc5ecf0962f5ab9f4126febf7ce33. 2023-07-22T12:35:02,682 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing fc6cb6b064d3fb792c9e01da5e382760, disabling compactions & flushes 2023-07-22T12:35:02,683 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1690029077151.fc6cb6b064d3fb792c9e01da5e382760. 2023-07-22T12:35:02,683 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1690029077151.fc6cb6b064d3fb792c9e01da5e382760. 2023-07-22T12:35:02,683 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1690029077151.fc6cb6b064d3fb792c9e01da5e382760. after waiting 0 ms 2023-07-22T12:35:02,683 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1690029077151.fc6cb6b064d3fb792c9e01da5e382760. 2023-07-22T12:35:02,683 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing fc6cb6b064d3fb792c9e01da5e382760 2/2 column families, dataSize=5.66 KB heapSize=8.50 KB 2023-07-22T12:35:02,696 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system_bulk/fc6cb6b064d3fb792c9e01da5e382760/.tmp/meta/2e526d0a19dc40b6b870ef1ff4dfcbcd is 323, key is bulk:ns2:test-16900290656251:01e992ceb654b1e583b990da186e85a8:d63801d92d8645ef95d1060b6ba3bbd9_SeqId_4_/meta:path/1690029234811/Put/seqid=0 2023-07-22T12:35:02,745 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2023-07-22T12:35:02,745 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2023-07-22T12:35:02,791 DEBUG [RS:0;jenkins-hbase3:43639 {}] regionserver.HRegionServer(1609): Waiting on 01e992ceb654b1e583b990da186e85a8, 1588230740, 595143905c5328f814fba73ad7017c76, 7e12e84cd2aa3dccee1c81fd5a1ead69, baf4f0d945c66abea73108272203bc6e, fc6cb6b064d3fb792c9e01da5e382760 2023-07-22T12:35:02,991 DEBUG [RS:0;jenkins-hbase3:43639 {}] regionserver.HRegionServer(1609): Waiting on 01e992ceb654b1e583b990da186e85a8, 1588230740, 595143905c5328f814fba73ad7017c76, 7e12e84cd2aa3dccee1c81fd5a1ead69, baf4f0d945c66abea73108272203bc6e, fc6cb6b064d3fb792c9e01da5e382760 2023-07-22T12:35:03,080 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.13 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/hbase/meta/1588230740/.tmp/rep_barrier/e06b637a18df4b08a2ac3e4952bee84f 2023-07-22T12:35:03,087 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: e06b637a18df4b08a2ac3e4952bee84f 2023-07-22T12:35:03,099 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/hbase/meta/1588230740/.tmp/table/6460b2006d8046ba88a58352c0880157 is 101, key is ns2:test-16900290656251,,1690029068374.0112d97ae6696b92341dfd7ff8824a0a./table:/1690029096060/DeleteFamily/seqid=0 2023-07-22T12:35:03,100 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.66 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system_bulk/fc6cb6b064d3fb792c9e01da5e382760/.tmp/meta/2e526d0a19dc40b6b870ef1ff4dfcbcd 2023-07-22T12:35:03,104 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.56 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/hbase/meta/1588230740/.tmp/table/6460b2006d8046ba88a58352c0880157 2023-07-22T12:35:03,110 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 2e526d0a19dc40b6b870ef1ff4dfcbcd 2023-07-22T12:35:03,114 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system_bulk/fc6cb6b064d3fb792c9e01da5e382760/.tmp/meta/2e526d0a19dc40b6b870ef1ff4dfcbcd as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system_bulk/fc6cb6b064d3fb792c9e01da5e382760/meta/2e526d0a19dc40b6b870ef1ff4dfcbcd 2023-07-22T12:35:03,115 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 6460b2006d8046ba88a58352c0880157 2023-07-22T12:35:03,117 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/hbase/meta/1588230740/.tmp/info/32c7f6f276b9434e98e87e5bef319158 as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/hbase/meta/1588230740/info/32c7f6f276b9434e98e87e5bef319158 2023-07-22T12:35:03,120 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system_bulk/fc6cb6b064d3fb792c9e01da5e382760/meta/2e526d0a19dc40b6b870ef1ff4dfcbcd, entries=16, sequenceid=12, filesize=8.1 K 2023-07-22T12:35:03,121 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~5.66 KB/5800, heapSize ~8.23 KB/8432, currentSize=0 B/0 for fc6cb6b064d3fb792c9e01da5e382760 in 438ms, sequenceid=12, compaction requested=false 2023-07-22T12:35:03,123 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/hbase/meta/1588230740/info/32c7f6f276b9434e98e87e5bef319158, entries=94, sequenceid=183, filesize=16.4 K 2023-07-22T12:35:03,123 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/hbase/meta/1588230740/.tmp/rep_barrier/e06b637a18df4b08a2ac3e4952bee84f as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/hbase/meta/1588230740/rep_barrier/e06b637a18df4b08a2ac3e4952bee84f 2023-07-22T12:35:03,128 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/hbase/meta/1588230740/rep_barrier/e06b637a18df4b08a2ac3e4952bee84f, entries=12, sequenceid=183, filesize=6.5 K 2023-07-22T12:35:03,129 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/hbase/meta/1588230740/.tmp/table/6460b2006d8046ba88a58352c0880157 as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/hbase/meta/1588230740/table/6460b2006d8046ba88a58352c0880157 2023-07-22T12:35:03,134 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/hbase/meta/1588230740/table/6460b2006d8046ba88a58352c0880157, entries=32, sequenceid=183, filesize=7.3 K 2023-07-22T12:35:03,135 DEBUG [StoreFileCloser-fc6cb6b064d3fb792c9e01da5e382760-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 2e526d0a19dc40b6b870ef1ff4dfcbcd 2023-07-22T12:35:03,138 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3022): Finished flush of dataSize ~38.00 KB/38910, heapSize ~66.25 KB/67840, currentSize=0 B/0 for 1588230740 in 948ms, sequenceid=183, compaction requested=false 2023-07-22T12:35:03,139 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 32c7f6f276b9434e98e87e5bef319158 2023-07-22T12:35:03,145 DEBUG [StoreFileCloser-1588230740-rep_barrier-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: e06b637a18df4b08a2ac3e4952bee84f 2023-07-22T12:35:03,147 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 6460b2006d8046ba88a58352c0880157 2023-07-22T12:35:03,149 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/backup/system_bulk/fc6cb6b064d3fb792c9e01da5e382760/recovered.edits/15.seqid, newMaxSeqId=15, maxSeqId=1 2023-07-22T12:35:03,149 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:35:03,150 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system_bulk,,1690029077151.fc6cb6b064d3fb792c9e01da5e382760. 2023-07-22T12:35:03,150 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for fc6cb6b064d3fb792c9e01da5e382760: 2023-07-22T12:35:03,150 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system_bulk,,1690029077151.fc6cb6b064d3fb792c9e01da5e382760. 2023-07-22T12:35:03,150 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 7e12e84cd2aa3dccee1c81fd5a1ead69, disabling compactions & flushes 2023-07-22T12:35:03,150 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns3:test-16900290656252,,1690029235139.7e12e84cd2aa3dccee1c81fd5a1ead69. 2023-07-22T12:35:03,150 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16900290656252,,1690029235139.7e12e84cd2aa3dccee1c81fd5a1ead69. 2023-07-22T12:35:03,150 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16900290656252,,1690029235139.7e12e84cd2aa3dccee1c81fd5a1ead69. after waiting 0 ms 2023-07-22T12:35:03,150 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16900290656252,,1690029235139.7e12e84cd2aa3dccee1c81fd5a1ead69. 2023-07-22T12:35:03,153 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/hbase/meta/1588230740/recovered.edits/186.seqid, newMaxSeqId=186, maxSeqId=1 2023-07-22T12:35:03,153 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:35:03,153 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-07-22T12:35:03,154 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-07-22T12:35:03,154 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-07-22T12:35:03,154 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2023-07-22T12:35:03,154 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns3/test-16900290656252/7e12e84cd2aa3dccee1c81fd5a1ead69/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2023-07-22T12:35:03,155 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:35:03,155 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns3:test-16900290656252,,1690029235139.7e12e84cd2aa3dccee1c81fd5a1ead69. 2023-07-22T12:35:03,155 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 7e12e84cd2aa3dccee1c81fd5a1ead69: 2023-07-22T12:35:03,155 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns3:test-16900290656252,,1690029235139.7e12e84cd2aa3dccee1c81fd5a1ead69. 2023-07-22T12:35:03,157 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 595143905c5328f814fba73ad7017c76, disabling compactions & flushes 2023-07-22T12:35:03,157 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:namespace,,1690029059700.595143905c5328f814fba73ad7017c76. 2023-07-22T12:35:03,157 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1690029059700.595143905c5328f814fba73ad7017c76. 2023-07-22T12:35:03,157 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1690029059700.595143905c5328f814fba73ad7017c76. after waiting 0 ms 2023-07-22T12:35:03,157 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1690029059700.595143905c5328f814fba73ad7017c76. 2023-07-22T12:35:03,158 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 595143905c5328f814fba73ad7017c76 1/1 column families, dataSize=249 B heapSize=1.02 KB 2023-07-22T12:35:03,171 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/hbase/namespace/595143905c5328f814fba73ad7017c76/.tmp/info/9539c528bfef4eb8bf6429368b70873e is 45, key is default/info:d/1690029061087/Put/seqid=0 2023-07-22T12:35:03,191 INFO [RS:0;jenkins-hbase3:43639 {}] regionserver.HRegionServer(1579): Waiting on 3 regions to close 2023-07-22T12:35:03,191 DEBUG [RS:0;jenkins-hbase3:43639 {}] regionserver.HRegionServer(1583): Online Regions={595143905c5328f814fba73ad7017c76=hbase:namespace,,1690029059700.595143905c5328f814fba73ad7017c76., baf4f0d945c66abea73108272203bc6e=test-1690029065625,,1690029166757.baf4f0d945c66abea73108272203bc6e., 01e992ceb654b1e583b990da186e85a8=ns2:test-16900290656251,,1690029231061.01e992ceb654b1e583b990da186e85a8.} 2023-07-22T12:35:03,191 DEBUG [RS:0;jenkins-hbase3:43639 {}] regionserver.HRegionServer(1609): Waiting on 01e992ceb654b1e583b990da186e85a8, 595143905c5328f814fba73ad7017c76, baf4f0d945c66abea73108272203bc6e 2023-07-22T12:35:03,225 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-07-22T12:35:03,225 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-16900290656253 2023-07-22T12:35:03,225 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-07-22T12:35:03,225 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2023-07-22T12:35:03,225 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2023-07-22T12:35:03,225 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-16900290656252 2023-07-22T12:35:03,392 DEBUG [RS:0;jenkins-hbase3:43639 {}] regionserver.HRegionServer(1609): Waiting on 01e992ceb654b1e583b990da186e85a8, 595143905c5328f814fba73ad7017c76, baf4f0d945c66abea73108272203bc6e 2023-07-22T12:35:03,576 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/hbase/namespace/595143905c5328f814fba73ad7017c76/.tmp/info/9539c528bfef4eb8bf6429368b70873e 2023-07-22T12:35:03,580 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 9539c528bfef4eb8bf6429368b70873e 2023-07-22T12:35:03,581 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/hbase/namespace/595143905c5328f814fba73ad7017c76/.tmp/info/9539c528bfef4eb8bf6429368b70873e as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/hbase/namespace/595143905c5328f814fba73ad7017c76/info/9539c528bfef4eb8bf6429368b70873e 2023-07-22T12:35:03,585 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/hbase/namespace/595143905c5328f814fba73ad7017c76/info/9539c528bfef4eb8bf6429368b70873e, entries=7, sequenceid=11, filesize=5.1 K 2023-07-22T12:35:03,586 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~249 B/249, heapSize ~1.01 KB/1032, currentSize=0 B/0 for 595143905c5328f814fba73ad7017c76 in 429ms, sequenceid=11, compaction requested=false 2023-07-22T12:35:03,589 DEBUG [StoreFileCloser-595143905c5328f814fba73ad7017c76-info-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 9539c528bfef4eb8bf6429368b70873e 2023-07-22T12:35:03,592 DEBUG [RS:0;jenkins-hbase3:43639 {}] regionserver.HRegionServer(1609): Waiting on 01e992ceb654b1e583b990da186e85a8, 595143905c5328f814fba73ad7017c76, baf4f0d945c66abea73108272203bc6e 2023-07-22T12:35:03,593 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/hbase/namespace/595143905c5328f814fba73ad7017c76/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2023-07-22T12:35:03,594 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:35:03,595 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:namespace,,1690029059700.595143905c5328f814fba73ad7017c76. 2023-07-22T12:35:03,595 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 595143905c5328f814fba73ad7017c76: 2023-07-22T12:35:03,595 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1690029059700.595143905c5328f814fba73ad7017c76. 2023-07-22T12:35:03,595 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing baf4f0d945c66abea73108272203bc6e, disabling compactions & flushes 2023-07-22T12:35:03,595 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region test-1690029065625,,1690029166757.baf4f0d945c66abea73108272203bc6e. 2023-07-22T12:35:03,595 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1690029065625,,1690029166757.baf4f0d945c66abea73108272203bc6e. 2023-07-22T12:35:03,595 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on test-1690029065625,,1690029166757.baf4f0d945c66abea73108272203bc6e. after waiting 0 ms 2023-07-22T12:35:03,595 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region test-1690029065625,,1690029166757.baf4f0d945c66abea73108272203bc6e. 2023-07-22T12:35:03,595 DEBUG [StoreFileCloser-baf4f0d945c66abea73108272203bc6e-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 28a90c4c400f435e9fc04a59e5431a5a_SeqId_4_ 2023-07-22T12:35:03,598 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/default/test-1690029065625/baf4f0d945c66abea73108272203bc6e/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2023-07-22T12:35:03,599 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:35:03,600 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed test-1690029065625,,1690029166757.baf4f0d945c66abea73108272203bc6e. 2023-07-22T12:35:03,600 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for baf4f0d945c66abea73108272203bc6e: 2023-07-22T12:35:03,600 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test-1690029065625,,1690029166757.baf4f0d945c66abea73108272203bc6e. 2023-07-22T12:35:03,600 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 01e992ceb654b1e583b990da186e85a8, disabling compactions & flushes 2023-07-22T12:35:03,600 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns2:test-16900290656251,,1690029231061.01e992ceb654b1e583b990da186e85a8. 2023-07-22T12:35:03,600 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16900290656251,,1690029231061.01e992ceb654b1e583b990da186e85a8. 2023-07-22T12:35:03,600 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16900290656251,,1690029231061.01e992ceb654b1e583b990da186e85a8. after waiting 0 ms 2023-07-22T12:35:03,600 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16900290656251,,1690029231061.01e992ceb654b1e583b990da186e85a8. 2023-07-22T12:35:03,602 DEBUG [StoreFileCloser-01e992ceb654b1e583b990da186e85a8-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: d63801d92d8645ef95d1060b6ba3bbd9_SeqId_4_ 2023-07-22T12:35:03,605 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/data/ns2/test-16900290656251/01e992ceb654b1e583b990da186e85a8/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2023-07-22T12:35:03,605 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:35:03,606 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns2:test-16900290656251,,1690029231061.01e992ceb654b1e583b990da186e85a8. 2023-07-22T12:35:03,606 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 01e992ceb654b1e583b990da186e85a8: 2023-07-22T12:35:03,606 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:test-16900290656251,,1690029231061.01e992ceb654b1e583b990da186e85a8. 2023-07-22T12:35:03,792 INFO [RS:0;jenkins-hbase3:43639 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,43639,1690029056327; all regions closed. 2023-07-22T12:35:03,804 DEBUG [RS:0;jenkins-hbase3:43639 {}] wal.AbstractFSWAL(1062): Moved 10 WAL file(s) to /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/oldWALs 2023-07-22T12:35:03,805 INFO [RS:0;jenkins-hbase3:43639 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C43639%2C1690029056327.meta:.meta(num 1690029290036) 2023-07-22T12:35:03,813 DEBUG [RS:0;jenkins-hbase3:43639 {}] wal.AbstractFSWAL(1062): Moved 7 WAL file(s) to /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/oldWALs 2023-07-22T12:35:03,813 INFO [RS:0;jenkins-hbase3:43639 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C43639%2C1690029056327:(num 1690029290060) 2023-07-22T12:35:03,813 DEBUG [RS:0;jenkins-hbase3:43639 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:35:03,813 INFO [RS:0;jenkins-hbase3:43639 {}] regionserver.LeaseManager(133): Closed leases 2023-07-22T12:35:03,814 INFO [RS:0;jenkins-hbase3:43639 {}] hbase.ChoreService(370): Chore service for: regionserver/jenkins-hbase3:0 had [ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2023-07-22T12:35:03,814 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-07-22T12:35:03,816 INFO [RS:0;jenkins-hbase3:43639 {}] ipc.NettyRpcServer(213): Stopping server on /172.31.12.81:43639 2023-07-22T12:35:03,826 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [jenkins-hbase3.apache.org,43639,1690029056327] 2023-07-22T12:35:03,827 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing jenkins-hbase3.apache.org,43639,1690029056327; numProcessing=1 2023-07-22T12:35:03,828 INFO [RegionServerTracker-0 {}] master.ServerManager(569): Cluster shutdown set; jenkins-hbase3.apache.org,43639,1690029056327 expired; onlineServers=0 2023-07-22T12:35:03,828 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2513): ***** STOPPING region server 'jenkins-hbase3.apache.org,37923,1690029053050' ***** 2023-07-22T12:35:03,828 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2527): STOPPED: Cluster shutdown set; onlineServer=0 2023-07-22T12:35:03,829 DEBUG [M:0;jenkins-hbase3:37923 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76ff47d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-07-22T12:35:03,829 INFO [M:0;jenkins-hbase3:37923 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-07-22T12:35:03,874 INFO [M:0;jenkins-hbase3:37923 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@2baba16b{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-07-22T12:35:03,875 INFO [M:0;jenkins-hbase3:37923 {}] server.AbstractConnector(383): Stopped ServerConnector@4e8aff63{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-07-22T12:35:03,875 INFO [M:0;jenkins-hbase3:37923 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-07-22T12:35:03,876 INFO [M:0;jenkins-hbase3:37923 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@36677652{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-07-22T12:35:03,876 INFO [M:0;jenkins-hbase3:37923 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@6ffeb5f7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/hadoop.log.dir/,STOPPED} 2023-07-22T12:35:03,877 INFO [M:0;jenkins-hbase3:37923 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,37923,1690029053050 2023-07-22T12:35:03,877 INFO [M:0;jenkins-hbase3:37923 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,37923,1690029053050; all regions closed. 2023-07-22T12:35:03,877 DEBUG [M:0;jenkins-hbase3:37923 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:35:03,877 INFO [M:0;jenkins-hbase3:37923 {}] master.HMaster(1664): Stopping master jetty server 2023-07-22T12:35:03,878 INFO [M:0;jenkins-hbase3:37923 {}] server.AbstractConnector(383): Stopped ServerConnector@6b055907{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-07-22T12:35:03,878 DEBUG [M:0;jenkins-hbase3:37923 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2023-07-22T12:35:03,878 INFO [M:0;jenkins-hbase3:37923 {}] master.BackupLogCleaner(171): Stopping BackupLogCleaner 2023-07-22T12:35:03,878 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2023-07-22T12:35:03,878 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1690029058566 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1690029058566,5,FailOnTimeoutGroup] 2023-07-22T12:35:03,878 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1690029058567 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1690029058567,5,FailOnTimeoutGroup] 2023-07-22T12:35:03,878 DEBUG [M:0;jenkins-hbase3:37923 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:35:03,879 DEBUG [M:0;jenkins-hbase3:37923 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2023-07-22T12:35:03,880 INFO [M:0;jenkins-hbase3:37923 {}] hbase.ChoreService(370): Chore service for: master/jenkins-hbase3:0 had [] on shutdown 2023-07-22T12:35:03,880 DEBUG [M:0;jenkins-hbase3:37923 {}] master.HMaster(1687): Stopping service threads 2023-07-22T12:35:03,880 INFO [M:0;jenkins-hbase3:37923 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2023-07-22T12:35:03,880 ERROR [M:0;jenkins-hbase3:37923 {}] procedure2.ProcedureExecutor(655): ThreadGroup java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] contains running threads; null: See STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-12,5,PEWorkerGroup] 2023-07-22T12:35:03,881 INFO [M:0;jenkins-hbase3:37923 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2023-07-22T12:35:03,881 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2023-07-22T12:35:03,881 WARN [M:0;jenkins-hbase3:37923 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2023-07-22T12:35:03,881 INFO [M:0;jenkins-hbase3:37923 {}] assignment.AssignmentManager(382): Stopping assignment manager 2023-07-22T12:35:03,881 INFO [M:0;jenkins-hbase3:37923 {}] region.MasterRegion(178): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2023-07-22T12:35:03,881 DEBUG [M:0;jenkins-hbase3:37923 {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-07-22T12:35:03,893 INFO [M:0;jenkins-hbase3:37923 {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:35:03,893 DEBUG [M:0;jenkins-hbase3:37923 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:35:03,893 DEBUG [M:0;jenkins-hbase3:37923 {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-07-22T12:35:03,893 DEBUG [M:0;jenkins-hbase3:37923 {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:35:03,893 INFO [M:0;jenkins-hbase3:37923 {}] regionserver.HRegion(2819): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=621.81 KB heapSize=756.46 KB 2023-07-22T12:35:03,905 DEBUG [M:0;jenkins-hbase3:37923 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/eacfdc4418904a18b10737006634fa5a is 88, key is hbase:meta,,1/info:sn/1690029059027/Put/seqid=0 2023-07-22T12:35:03,909 INFO [M:0;jenkins-hbase3:37923 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=530 B at sequenceid=1616 (bloomFilter=true), to=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/eacfdc4418904a18b10737006634fa5a 2023-07-22T12:35:03,913 DEBUG [M:0;jenkins-hbase3:37923 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: eacfdc4418904a18b10737006634fa5a 2023-07-22T12:35:03,926 INFO [RS:0;jenkins-hbase3:43639 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,43639,1690029056327; zookeeper connection closed. 2023-07-22T12:35:03,926 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@107d38c0 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@107d38c0 2023-07-22T12:35:03,927 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2023-07-22T12:35:03,927 DEBUG [M:0;jenkins-hbase3:37923 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4f1d5012392347fbb2c6521abce1435e is 988, key is \x00\x00\x00\x00\x00\x00\x00\x1D/proc:d/1690029077940/Put/seqid=0 2023-07-22T12:35:04,331 INFO [M:0;jenkins-hbase3:37923 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=621.21 KB at sequenceid=1616 (bloomFilter=true), to=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4f1d5012392347fbb2c6521abce1435e 2023-07-22T12:35:04,336 INFO [M:0;jenkins-hbase3:37923 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4f1d5012392347fbb2c6521abce1435e 2023-07-22T12:35:04,336 DEBUG [M:0;jenkins-hbase3:37923 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 4f1d5012392347fbb2c6521abce1435e 2023-07-22T12:35:04,348 DEBUG [M:0;jenkins-hbase3:37923 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2fb7930e76f54568a7fb9592608ba7fb is 82, key is jenkins-hbase3.apache.org,43639,1690029056327/rs:state/1690029058661/Put/seqid=0 2023-07-22T12:35:04,752 INFO [M:0;jenkins-hbase3:37923 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=1616 (bloomFilter=true), to=hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2fb7930e76f54568a7fb9592608ba7fb 2023-07-22T12:35:04,757 DEBUG [M:0;jenkins-hbase3:37923 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 2fb7930e76f54568a7fb9592608ba7fb 2023-07-22T12:35:04,757 DEBUG [M:0;jenkins-hbase3:37923 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/eacfdc4418904a18b10737006634fa5a as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/eacfdc4418904a18b10737006634fa5a 2023-07-22T12:35:04,762 INFO [M:0;jenkins-hbase3:37923 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/eacfdc4418904a18b10737006634fa5a, entries=8, sequenceid=1616, filesize=5.5 K 2023-07-22T12:35:04,763 DEBUG [M:0;jenkins-hbase3:37923 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4f1d5012392347fbb2c6521abce1435e as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4f1d5012392347fbb2c6521abce1435e 2023-07-22T12:35:04,767 INFO [M:0;jenkins-hbase3:37923 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4f1d5012392347fbb2c6521abce1435e 2023-07-22T12:35:04,767 INFO [M:0;jenkins-hbase3:37923 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4f1d5012392347fbb2c6521abce1435e, entries=173, sequenceid=1616, filesize=39.1 K 2023-07-22T12:35:04,768 DEBUG [M:0;jenkins-hbase3:37923 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2fb7930e76f54568a7fb9592608ba7fb as hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2fb7930e76f54568a7fb9592608ba7fb 2023-07-22T12:35:04,772 INFO [M:0;jenkins-hbase3:37923 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34745/user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2fb7930e76f54568a7fb9592608ba7fb, entries=1, sequenceid=1616, filesize=5.1 K 2023-07-22T12:35:04,773 INFO [M:0;jenkins-hbase3:37923 {}] regionserver.HRegion(3022): Finished flush of dataSize ~621.81 KB/636732, heapSize ~756.16 KB/774312, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 880ms, sequenceid=1616, compaction requested=false 2023-07-22T12:35:04,775 DEBUG [StoreFileCloser-1595e783b53d99cd5eef43b6debb2682-info-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: eacfdc4418904a18b10737006634fa5a 2023-07-22T12:35:04,778 DEBUG [StoreFileCloser-1595e783b53d99cd5eef43b6debb2682-proc-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 4f1d5012392347fbb2c6521abce1435e 2023-07-22T12:35:04,780 DEBUG [StoreFileCloser-1595e783b53d99cd5eef43b6debb2682-rs-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 2fb7930e76f54568a7fb9592608ba7fb 2023-07-22T12:35:04,780 INFO [M:0;jenkins-hbase3:37923 {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:35:04,780 DEBUG [M:0;jenkins-hbase3:37923 {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-07-22T12:35:04,783 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/b8189408-8905-db9b-faf1-c15e0fe3f947/MasterData/WALs/jenkins-hbase3.apache.org,37923,1690029053050/jenkins-hbase3.apache.org%2C37923%2C1690029053050.1690029057738 not finished, retry = 0 2023-07-22T12:35:04,884 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-07-22T12:35:04,884 INFO [M:0;jenkins-hbase3:37923 {}] flush.MasterFlushTableProcedureManager(81): stop: server shutting down. 2023-07-22T12:35:04,884 INFO [M:0;jenkins-hbase3:37923 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2023-07-22T12:35:04,885 INFO [M:0;jenkins-hbase3:37923 {}] ipc.NettyRpcServer(213): Stopping server on /172.31.12.81:37923 2023-07-22T12:35:04,989 INFO [M:0;jenkins-hbase3:37923 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,37923,1690029053050; zookeeper connection closed. 2023-07-22T12:35:04,999 WARN [Listener at jenkins-hbase3.apache.org/43645 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-07-22T12:35:05,009 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-07-22T12:35:05,117 WARN [BP-821173014-172.31.12.81-1690029049712 heartbeating to localhost/127.0.0.1:34745 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-07-22T12:35:05,117 WARN [BP-821173014-172.31.12.81-1690029049712 heartbeating to localhost/127.0.0.1:34745 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-821173014-172.31.12.81-1690029049712 (Datanode Uuid 781397b1-b4c2-433c-adfa-c2f66f3a63ff) service to localhost/127.0.0.1:34745 2023-07-22T12:35:05,119 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/cluster_3df5f3c2-318c-5fd6-9f69-2531a0a95eef/dfs/data/data1/current/BP-821173014-172.31.12.81-1690029049712 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-07-22T12:35:05,120 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1b0e3d3b-c393-fee3-aab2-03d16662ced7/cluster_3df5f3c2-318c-5fd6-9f69-2531a0a95eef/dfs/data/data2/current/BP-821173014-172.31.12.81-1690029049712 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-07-22T12:35:05,152 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-07-22T12:35:05,375 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2023-07-22T12:35:05,375 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.HBaseTestingUtility(2857): Stopping mini mapreduce cluster... 2023-07-22T12:35:05,385 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-07-22T12:35:05,499 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-07-22T12:35:05,608 ERROR [Thread[Thread-275,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-07-22T12:35:05,609 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-07-22T12:35:05,714 WARN [ApplicationMaster Launcher {}] amlauncher.ApplicationMasterLauncher$LauncherThread(122): org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher$LauncherThread interrupted. Returning. 2023-07-22T12:35:05,721 ERROR [SchedulerEventDispatcher:Event Processor {}] event.EventDispatcher$EventProcessor(61): Returning, interrupted : java.lang.InterruptedException 2023-07-22T12:35:05,722 ERROR [Thread[Thread-285,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-07-22T12:35:05,727 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-07-22T12:35:05,829 ERROR [Thread[Thread-250,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-07-22T12:35:05,829 INFO [Listener at jenkins-hbase3.apache.org/43645 {}] hbase.HBaseTestingUtility(2860): Mini mapreduce cluster stopped