2023-08-10 06:27:31,465 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@175c2241 2023-08-10 06:27:31,478 main DEBUG Took 0.011055 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2023-08-10 06:27:31,478 main DEBUG PluginManager 'Core' found 129 plugins 2023-08-10 06:27:31,479 main DEBUG PluginManager 'Level' found 0 plugins 2023-08-10 06:27:31,480 main DEBUG PluginManager 'Lookup' found 16 plugins 2023-08-10 06:27:31,481 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:27:31,489 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2023-08-10 06:27:31,501 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:27:31,502 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:27:31,503 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:27:31,503 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:27:31,504 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:27:31,504 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:27:31,505 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:27:31,505 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:27:31,505 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:27:31,505 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:27:31,506 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:27:31,506 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:27:31,507 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:27:31,507 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:27:31,507 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:27:31,507 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:27:31,508 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:27:31,508 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:27:31,508 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:27:31,509 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:27:31,509 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:27:31,509 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:27:31,510 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:27:31,510 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:27:31,510 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:27:31,510 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2023-08-10 06:27:31,512 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:27:31,514 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2023-08-10 06:27:31,516 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2023-08-10 06:27:31,516 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2023-08-10 06:27:31,518 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2023-08-10 06:27:31,518 main DEBUG PluginManager 'Converter' found 47 plugins 2023-08-10 06:27:31,531 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2023-08-10 06:27:31,533 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2023-08-10 06:27:31,534 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2023-08-10 06:27:31,534 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2023-08-10 06:27:31,535 main DEBUG createAppenders(={Console}) 2023-08-10 06:27:31,536 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@175c2241 initialized 2023-08-10 06:27:31,536 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@175c2241 2023-08-10 06:27:31,536 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@175c2241 OK. 2023-08-10 06:27:31,537 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2023-08-10 06:27:31,537 main DEBUG OutputStream closed 2023-08-10 06:27:31,537 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2023-08-10 06:27:31,537 main DEBUG Appender DefaultConsole-1 stopped with status true 2023-08-10 06:27:31,538 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@2145b572 OK 2023-08-10 06:27:31,581 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586 2023-08-10 06:27:31,583 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=StatusLogger 2023-08-10 06:27:31,584 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=ContextSelector 2023-08-10 06:27:31,585 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name= 2023-08-10 06:27:31,585 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.directory 2023-08-10 06:27:31,586 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2023-08-10 06:27:31,586 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.zookeeper 2023-08-10 06:27:31,586 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2023-08-10 06:27:31,587 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2023-08-10 06:27:31,587 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2023-08-10 06:27:31,587 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase 2023-08-10 06:27:31,587 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop 2023-08-10 06:27:31,588 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2023-08-10 06:27:31,588 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2023-08-10 06:27:31,588 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2023-08-10 06:27:31,588 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2023-08-10 06:27:31,589 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2023-08-10 06:27:31,590 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Appenders,name=Console 2023-08-10 06:27:31,593 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-08-10 06:27:31,593 main DEBUG Reconfiguration complete for context[name=1b6d3586] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.6.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@49912c99) with optional ClassLoader: null 2023-08-10 06:27:31,593 main DEBUG Shutdown hook enabled. Registering a new one. 2023-08-10 06:27:31,594 main DEBUG LoggerContext[name=1b6d3586, org.apache.logging.log4j.core.LoggerContext@49912c99] started OK. 2023-08-10T06:27:31,628 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.backup.TestFullRestore timeout: 13 mins 2023-08-10 06:27:31,634 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2023-08-10 06:27:31,634 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-08-10T06:27:31,980 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc 2023-08-10T06:27:31,985 DEBUG [Time-limited test {}] impl.BackupManager(127): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner. Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager.Added master procedure manager: org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-08-10T06:27:31,987 DEBUG [Time-limited test {}] impl.BackupManager(157): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager. Added region observer: org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:27:31,988 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2023-08-10T06:27:31,992 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/cluster_28117de4-e82e-18fe-7324-f4a37a8c1f1e, deleteOnExit=true 2023-08-10T06:27:31,992 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2023-08-10T06:27:31,992 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/test.cache.data in system properties and HBase conf 2023-08-10T06:27:31,993 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/hadoop.tmp.dir in system properties and HBase conf 2023-08-10T06:27:31,993 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/hadoop.log.dir in system properties and HBase conf 2023-08-10T06:27:31,993 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir in system properties and HBase conf 2023-08-10T06:27:31,994 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-08-10T06:27:31,994 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-08-10T06:27:32,116 WARN [Time-limited test {}] util.NativeCodeLoader(62): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2023-08-10T06:27:32,607 DEBUG [Time-limited test {}] fs.HFileSystem(308): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-08-10T06:27:32,613 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-08-10T06:27:32,613 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-08-10T06:27:32,614 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-08-10T06:27:32,614 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-08-10T06:27:32,615 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-08-10T06:27:32,615 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-08-10T06:27:32,616 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-08-10T06:27:32,616 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/dfs.journalnode.edits.dir in system properties and HBase conf 2023-08-10T06:27:32,617 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-08-10T06:27:32,617 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/nfs.dump.dir in system properties and HBase conf 2023-08-10T06:27:32,618 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/java.io.tmpdir in system properties and HBase conf 2023-08-10T06:27:32,618 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/dfs.journalnode.edits.dir in system properties and HBase conf 2023-08-10T06:27:32,619 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-08-10T06:27:32,619 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2023-08-10T06:27:33,052 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-08-10T06:27:33,056 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-08-10T06:27:33,362 WARN [Time-limited test {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2023-08-10T06:27:33,500 INFO [Time-limited test {}] log.Slf4jLog(67): Logging to org.apache.logging.slf4j.Log4jLogger@4a06a2d5 via org.mortbay.log.Slf4jLog 2023-08-10T06:27:33,516 WARN [Time-limited test {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-08-10T06:27:33,556 INFO [Time-limited test {}] log.Slf4jLog(67): jetty-6.1.26 2023-08-10T06:27:33,593 INFO [Time-limited test {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/java.io.tmpdir/Jetty_localhost_43943_hdfs____.z23c1p/webapp 2023-08-10T06:27:33,731 INFO [Time-limited test {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:43943 2023-08-10T06:27:33,760 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-08-10T06:27:33,760 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-08-10T06:27:34,383 WARN [Listener at localhost/45379 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-08-10T06:27:34,461 WARN [Listener at localhost/45379 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-08-10T06:27:34,485 WARN [Listener at localhost/45379 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-08-10T06:27:34,494 INFO [Listener at localhost/45379 {}] log.Slf4jLog(67): jetty-6.1.26 2023-08-10T06:27:34,502 INFO [Listener at localhost/45379 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/java.io.tmpdir/Jetty_localhost_44963_datanode____.311tcz/webapp 2023-08-10T06:27:34,648 INFO [Listener at localhost/45379 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:44963 2023-08-10T06:27:34,985 WARN [Listener at localhost/36815 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-08-10T06:27:35,281 WARN [Thread-52 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-08-10T06:27:35,445 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xc7de024cabb67fd: Processing first storage report for DS-d83403a9-290e-4799-b60c-fa534525be24 from datanode 7c55e065-91e0-4898-9c2b-da8e30448879 2023-08-10T06:27:35,447 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xc7de024cabb67fd: from storage DS-d83403a9-290e-4799-b60c-fa534525be24 node DatanodeRegistration(127.0.0.1:43219, datanodeUuid=7c55e065-91e0-4898-9c2b-da8e30448879, infoPort=34095, infoSecurePort=0, ipcPort=36815, storageInfo=lv=-57;cid=testClusterID;nsid=474515990;c=1691648853138), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2023-08-10T06:27:35,447 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xc7de024cabb67fd: Processing first storage report for DS-54d4f494-15ca-4cac-88cf-94eb3350fe4a from datanode 7c55e065-91e0-4898-9c2b-da8e30448879 2023-08-10T06:27:35,447 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xc7de024cabb67fd: from storage DS-54d4f494-15ca-4cac-88cf-94eb3350fe4a node DatanodeRegistration(127.0.0.1:43219, datanodeUuid=7c55e065-91e0-4898-9c2b-da8e30448879, infoPort=34095, infoSecurePort=0, ipcPort=36815, storageInfo=lv=-57;cid=testClusterID;nsid=474515990;c=1691648853138), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-08-10T06:27:35,528 DEBUG [Listener at localhost/36815 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc 2023-08-10T06:27:35,624 INFO [Listener at localhost/36815 {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/cluster_28117de4-e82e-18fe-7324-f4a37a8c1f1e/zookeeper_0, clientPort=61704, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/cluster_28117de4-e82e-18fe-7324-f4a37a8c1f1e/zookeeper_0/version-2, dataDirSize=424 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/cluster_28117de4-e82e-18fe-7324-f4a37a8c1f1e/zookeeper_0/version-2, dataLogSize=424 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, serverId=0 2023-08-10T06:27:35,647 INFO [Listener at localhost/36815 {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=61704 2023-08-10T06:27:35,657 INFO [Listener at localhost/36815 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-08-10T06:27:35,660 INFO [Listener at localhost/36815 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-08-10T06:27:36,384 INFO [Listener at localhost/36815 {}] util.FSUtils(461): Created version file at hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e with version=8 2023-08-10T06:27:36,385 INFO [Listener at localhost/36815 {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging 2023-08-10T06:27:36,401 DEBUG [Listener at localhost/36815 {}] hbase.LocalHBaseCluster(134): Setting Master Port to random. 2023-08-10T06:27:36,401 DEBUG [Listener at localhost/36815 {}] hbase.LocalHBaseCluster(141): Setting RegionServer Port to random. 2023-08-10T06:27:36,401 DEBUG [Listener at localhost/36815 {}] hbase.LocalHBaseCluster(151): Setting RS InfoServer Port to random. 2023-08-10T06:27:36,401 DEBUG [Listener at localhost/36815 {}] hbase.LocalHBaseCluster(159): Setting Master InfoServer Port to random. 2023-08-10T06:27:36,611 DEBUG [Listener at localhost/36815 {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2023-08-10T06:27:36,767 INFO [Listener at localhost/36815 {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2023-08-10T06:27:37,437 INFO [Listener at localhost/36815 {}] client.ConnectionUtils(128): master/jenkins-hbase3:0 server-side Connection retries=45 2023-08-10T06:27:37,494 INFO [Listener at localhost/36815 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-08-10T06:27:37,495 INFO [Listener at localhost/36815 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-08-10T06:27:37,495 INFO [Listener at localhost/36815 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-08-10T06:27:37,496 INFO [Listener at localhost/36815 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-08-10T06:27:37,496 INFO [Listener at localhost/36815 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-08-10T06:27:37,654 INFO [Listener at localhost/36815 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-08-10T06:27:37,758 DEBUG [Listener at localhost/36815 {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2023-08-10T06:27:37,765 INFO [Listener at localhost/36815 {}] ipc.NettyRpcServer(177): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-08-10T06:27:37,821 DEBUG [Listener at localhost/36815 {}] channel.DefaultChannelId(79): -Dio.netty.processId: 8753 (auto-detected) 2023-08-10T06:27:37,822 DEBUG [Listener at localhost/36815 {}] channel.DefaultChannelId(101): -Dio.netty.machineId: 02:42:1c:ff:fe:e2:13:ea (auto-detected) 2023-08-10T06:27:37,860 INFO [Listener at localhost/36815 {}] ipc.NettyRpcServer(144): Bind to /172.31.12.81:33705 2023-08-10T06:27:37,875 INFO [Listener at localhost/36815 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-08-10T06:27:37,877 INFO [Listener at localhost/36815 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-08-10T06:27:37,894 INFO [Listener at localhost/36815 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=master:33705 connecting to ZooKeeper ensemble=127.0.0.1:61704 2023-08-10T06:27:37,940 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:337050x0, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-08-10T06:27:37,944 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): master:33705-0x10060ea6e8b0000 connected 2023-08-10T06:27:37,979 DEBUG [Listener at localhost/36815 {}] zookeeper.ZKUtil(113): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-08-10T06:27:37,984 DEBUG [Listener at localhost/36815 {}] zookeeper.ZKUtil(113): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-08-10T06:27:37,989 DEBUG [Listener at localhost/36815 {}] zookeeper.ZKUtil(113): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-08-10T06:27:38,003 DEBUG [Listener at localhost/36815 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33705 2023-08-10T06:27:38,004 DEBUG [Listener at localhost/36815 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33705 2023-08-10T06:27:38,004 DEBUG [Listener at localhost/36815 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33705 2023-08-10T06:27:38,006 DEBUG [Listener at localhost/36815 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33705 2023-08-10T06:27:38,007 DEBUG [Listener at localhost/36815 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33705 2023-08-10T06:27:38,063 INFO [Listener at localhost/36815 {}] log.Log(170): Logging initialized @7306ms to org.apache.hbase.thirdparty.org.eclipse.jetty.util.log.Slf4jLog 2023-08-10T06:27:38,201 INFO [Listener at localhost/36815 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-08-10T06:27:38,202 INFO [Listener at localhost/36815 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-08-10T06:27:38,203 INFO [Listener at localhost/36815 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-08-10T06:27:38,205 INFO [Listener at localhost/36815 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2023-08-10T06:27:38,205 INFO [Listener at localhost/36815 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-08-10T06:27:38,205 INFO [Listener at localhost/36815 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-08-10T06:27:38,209 INFO [Listener at localhost/36815 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-08-10T06:27:38,342 INFO [Listener at localhost/36815 {}] http.HttpServer(1219): Jetty bound to port 44311 2023-08-10T06:27:38,344 INFO [Listener at localhost/36815 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-08-10T06:27:38,378 INFO [Listener at localhost/36815 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-08-10T06:27:38,382 INFO [Listener at localhost/36815 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@71341aff{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/hadoop.log.dir/,AVAILABLE} 2023-08-10T06:27:38,382 INFO [Listener at localhost/36815 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-08-10T06:27:38,383 INFO [Listener at localhost/36815 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@3d08223b{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-08-10T06:27:38,455 INFO [Listener at localhost/36815 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-08-10T06:27:38,470 INFO [Listener at localhost/36815 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-08-10T06:27:38,470 INFO [Listener at localhost/36815 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-08-10T06:27:38,472 INFO [Listener at localhost/36815 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2023-08-10T06:27:38,481 INFO [Listener at localhost/36815 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-08-10T06:27:39,667 INFO [Listener at localhost/36815 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@3eb692b5{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-08-10T06:27:39,679 INFO [Listener at localhost/36815 {}] server.AbstractConnector(333): Started ServerConnector@1949aa43{HTTP/1.1, (http/1.1)}{0.0.0.0:44311} 2023-08-10T06:27:39,679 INFO [Listener at localhost/36815 {}] server.Server(415): Started @8922ms 2023-08-10T06:27:39,683 INFO [Listener at localhost/36815 {}] master.HMaster(484): hbase.rootdir=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e, hbase.cluster.distributed=false 2023-08-10T06:27:39,756 INFO [Listener at localhost/36815 {}] client.ConnectionUtils(128): regionserver/jenkins-hbase3:0 server-side Connection retries=45 2023-08-10T06:27:39,756 INFO [Listener at localhost/36815 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-08-10T06:27:39,757 INFO [Listener at localhost/36815 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-08-10T06:27:39,757 INFO [Listener at localhost/36815 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-08-10T06:27:39,757 INFO [Listener at localhost/36815 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-08-10T06:27:39,757 INFO [Listener at localhost/36815 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-08-10T06:27:39,769 INFO [Listener at localhost/36815 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-08-10T06:27:39,772 INFO [Listener at localhost/36815 {}] ipc.NettyRpcServer(177): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-08-10T06:27:39,774 INFO [Listener at localhost/36815 {}] ipc.NettyRpcServer(144): Bind to /172.31.12.81:45101 2023-08-10T06:27:39,777 INFO [Listener at localhost/36815 {}] hfile.BlockCacheFactory(127): Allocating BlockCache size=782.40 MB, blockSize=64 KB 2023-08-10T06:27:39,786 DEBUG [Listener at localhost/36815 {}] mob.MobFileCache(120): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2023-08-10T06:27:39,787 INFO [Listener at localhost/36815 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-08-10T06:27:39,790 INFO [Listener at localhost/36815 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-08-10T06:27:39,792 INFO [Listener at localhost/36815 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=regionserver:45101 connecting to ZooKeeper ensemble=127.0.0.1:61704 2023-08-10T06:27:39,797 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:451010x0, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-08-10T06:27:39,798 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): regionserver:45101-0x10060ea6e8b0001 connected 2023-08-10T06:27:39,800 DEBUG [Listener at localhost/36815 {}] zookeeper.ZKUtil(113): regionserver:45101-0x10060ea6e8b0001, quorum=127.0.0.1:61704, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-08-10T06:27:39,802 DEBUG [Listener at localhost/36815 {}] zookeeper.ZKUtil(113): regionserver:45101-0x10060ea6e8b0001, quorum=127.0.0.1:61704, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-08-10T06:27:39,803 DEBUG [Listener at localhost/36815 {}] zookeeper.ZKUtil(113): regionserver:45101-0x10060ea6e8b0001, quorum=127.0.0.1:61704, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-08-10T06:27:39,808 DEBUG [Listener at localhost/36815 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45101 2023-08-10T06:27:39,808 DEBUG [Listener at localhost/36815 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45101 2023-08-10T06:27:39,811 DEBUG [Listener at localhost/36815 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45101 2023-08-10T06:27:39,813 DEBUG [Listener at localhost/36815 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45101 2023-08-10T06:27:39,814 DEBUG [Listener at localhost/36815 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45101 2023-08-10T06:27:39,817 INFO [Listener at localhost/36815 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-08-10T06:27:39,817 INFO [Listener at localhost/36815 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-08-10T06:27:39,817 INFO [Listener at localhost/36815 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-08-10T06:27:39,819 INFO [Listener at localhost/36815 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2023-08-10T06:27:39,819 INFO [Listener at localhost/36815 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-08-10T06:27:39,819 INFO [Listener at localhost/36815 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-08-10T06:27:39,819 INFO [Listener at localhost/36815 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-08-10T06:27:39,821 INFO [Listener at localhost/36815 {}] http.HttpServer(1219): Jetty bound to port 40337 2023-08-10T06:27:39,822 INFO [Listener at localhost/36815 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-08-10T06:27:39,831 INFO [Listener at localhost/36815 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-08-10T06:27:39,832 INFO [Listener at localhost/36815 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@5d880175{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/hadoop.log.dir/,AVAILABLE} 2023-08-10T06:27:39,832 INFO [Listener at localhost/36815 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-08-10T06:27:39,833 INFO [Listener at localhost/36815 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@6266f472{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-08-10T06:27:39,851 INFO [Listener at localhost/36815 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-08-10T06:27:39,853 INFO [Listener at localhost/36815 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-08-10T06:27:39,853 INFO [Listener at localhost/36815 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-08-10T06:27:39,853 INFO [Listener at localhost/36815 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2023-08-10T06:27:39,859 INFO [Listener at localhost/36815 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-08-10T06:27:39,865 INFO [Listener at localhost/36815 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@71b422e2{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-08-10T06:27:39,867 INFO [Listener at localhost/36815 {}] server.AbstractConnector(333): Started ServerConnector@39545045{HTTP/1.1, (http/1.1)}{0.0.0.0:40337} 2023-08-10T06:27:39,867 INFO [Listener at localhost/36815 {}] server.Server(415): Started @9110ms 2023-08-10T06:27:39,874 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-08-10T06:27:39,887 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@644017ac{HTTP/1.1, (http/1.1)}{0.0.0.0:38457} 2023-08-10T06:27:39,888 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(415): Started @9132ms 2023-08-10T06:27:39,889 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2387): Adding backup master ZNode /1/backup-masters/jenkins-hbase3.apache.org,33705,1691648856595 2023-08-10T06:27:39,905 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-08-10T06:27:39,905 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45101-0x10060ea6e8b0001, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-08-10T06:27:39,908 DEBUG [M:0;jenkins-hbase3:33705 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;jenkins-hbase3:33705 2023-08-10T06:27:39,913 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Set watcher on existing znode=/1/backup-masters/jenkins-hbase3.apache.org,33705,1691648856595 2023-08-10T06:27:39,934 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-08-10T06:27:39,934 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45101-0x10060ea6e8b0001, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-08-10T06:27:39,935 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:27:39,935 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45101-0x10060ea6e8b0001, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:27:39,937 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Set watcher on existing znode=/1/master 2023-08-10T06:27:39,940 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /1/backup-masters/jenkins-hbase3.apache.org,33705,1691648856595 from backup master directory 2023-08-10T06:27:39,940 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Set watcher on existing znode=/1/master 2023-08-10T06:27:39,944 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/backup-masters/jenkins-hbase3.apache.org,33705,1691648856595 2023-08-10T06:27:39,944 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45101-0x10060ea6e8b0001, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-08-10T06:27:39,945 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-08-10T06:27:39,945 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-08-10T06:27:39,946 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=jenkins-hbase3.apache.org,33705,1691648856595 2023-08-10T06:27:39,950 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 352, initial count 0 2023-08-10T06:27:39,952 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 391, initial count 0 2023-08-10T06:27:40,094 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] util.FSUtils(610): Created cluster ID file at hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase.id with ID: e031a179-98a7-4aac-acc3-d4298684a610 2023-08-10T06:27:40,141 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-08-10T06:27:40,161 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45101-0x10060ea6e8b0001, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:27:40,161 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:27:40,230 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(355): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:27:40,233 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2023-08-10T06:27:40,261 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(270): ClientProtocol::create wrong number of arguments, should be hadoop 3.2 or below 2023-08-10T06:27:40,261 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(276): ClientProtocol::create wrong number of arguments, should be hadoop 2.x 2023-08-10T06:27:40,263 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(285): can not find SHOULD_REPLICATE flag, should be hadoop 2.x java.lang.IllegalArgumentException: No enum constant org.apache.hadoop.fs.CreateFlag.SHOULD_REPLICATE at java.lang.Enum.valueOf(Enum.java:238) ~[?:1.8.0_362] at org.apache.hadoop.fs.CreateFlag.valueOf(CreateFlag.java:63) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.loadShouldReplicateFlag(FanOutOneBlockAsyncDFSOutputHelper.java:283) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.(FanOutOneBlockAsyncDFSOutputHelper.java:310) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:140) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:950) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:573) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:570) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-08-10T06:27:40,278 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:141) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:950) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:573) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:570) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-08-10T06:27:40,280 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-08-10T06:27:40,362 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7093): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/MasterData/data/master/store 2023-08-10T06:27:40,789 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2023-08-10T06:27:40,790 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:27:40,790 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-08-10T06:27:40,790 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-08-10T06:27:40,790 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-08-10T06:27:40,791 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-08-10T06:27:40,791 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-08-10T06:27:40,791 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-08-10T06:27:40,791 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-08-10T06:27:40,793 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(232): failed to clean up initializing flag: hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/MasterData/data/master/store/.initializing 2023-08-10T06:27:40,794 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(202): WALDir=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/MasterData/WALs/jenkins-hbase3.apache.org,33705,1691648856595 2023-08-10T06:27:40,805 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-08-10T06:27:40,824 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C33705%2C1691648856595, suffix=, logDir=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/MasterData/WALs/jenkins-hbase3.apache.org,33705,1691648856595, archiveDir=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/MasterData/oldWALs, maxLogs=10 2023-08-10T06:27:40,853 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/MasterData/WALs/jenkins-hbase3.apache.org,33705,1691648856595/jenkins-hbase3.apache.org%2C33705%2C1691648856595.1691648860831, exclude list is [], retry=0 2023-08-10T06:27:40,889 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK] 2023-08-10T06:27:40,897 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(123): Hadoop 3.2 and below use unshaded protobuf. java.lang.ClassNotFoundException: org.apache.hadoop.thirdparty.protobuf.MessageLite at java.net.URLClassLoader.findClass(URLClassLoader.java:387) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:418) ~[?:1.8.0_362] at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ~[?:1.8.0_362] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.ProtobufDecoder.(ProtobufDecoder.java:118) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.processWriteBlockResponse(FanOutOneBlockAsyncDFSOutputHelper.java:346) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$100(FanOutOneBlockAsyncDFSOutputHelper.java:120) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$4.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:430) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:185) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:35) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hadoop.hbase.util.NettyFutureUtils.addListener(NettyFutureUtils.java:52) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:424) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$300(FanOutOneBlockAsyncDFSOutputHelper.java:120) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:482) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:477) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:653) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:691) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:489) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:397) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.4.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-08-10T06:27:40,982 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/MasterData/WALs/jenkins-hbase3.apache.org,33705,1691648856595/jenkins-hbase3.apache.org%2C33705%2C1691648856595.1691648860831 2023-08-10T06:27:40,983 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK]] 2023-08-10T06:27:40,984 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7254): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:27:40,984 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:27:40,989 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7296): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2023-08-10T06:27:40,990 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7299): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2023-08-10T06:27:41,073 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2023-08-10T06:27:41,119 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2023-08-10T06:27:41,125 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:27:41,129 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-08-10T06:27:41,130 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2023-08-10T06:27:41,133 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2023-08-10T06:27:41,133 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:27:41,134 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:27:41,135 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2023-08-10T06:27:41,138 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2023-08-10T06:27:41,138 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:27:41,139 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:27:41,139 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2023-08-10T06:27:41,142 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2023-08-10T06:27:41,142 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:27:41,144 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:27:41,149 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-08-10T06:27:41,151 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-08-10T06:27:41,170 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2023-08-10T06:27:41,175 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1076): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2023-08-10T06:27:41,180 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:27:41,181 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1093): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10392851840, jitterRate=-0.03209024667739868}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2023-08-10T06:27:41,187 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(992): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2023-08-10T06:27:41,189 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2023-08-10T06:27:41,224 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@707a81ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:27:41,283 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(859): No meta location available on zookeeper, skip migrating... 2023-08-10T06:27:41,307 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2023-08-10T06:27:41,307 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(564): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2023-08-10T06:27:41,310 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2023-08-10T06:27:41,312 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(584): Recovered RegionProcedureStore lease in 1 msec 2023-08-10T06:27:41,319 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(598): Loaded RegionProcedureStore in 6 msec 2023-08-10T06:27:41,319 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2023-08-10T06:27:41,356 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2023-08-10T06:27:41,375 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Unable to get data of znode /1/balancer because node does not exist (not necessarily an error) 2023-08-10T06:27:41,378 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/balancer already deleted, retry=false 2023-08-10T06:27:41,382 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(150): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2023-08-10T06:27:41,383 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Unable to get data of znode /1/normalizer because node does not exist (not necessarily an error) 2023-08-10T06:27:41,385 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/normalizer already deleted, retry=false 2023-08-10T06:27:41,388 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2023-08-10T06:27:41,395 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Unable to get data of znode /1/switch/split because node does not exist (not necessarily an error) 2023-08-10T06:27:41,397 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/split already deleted, retry=false 2023-08-10T06:27:41,398 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Unable to get data of znode /1/switch/merge because node does not exist (not necessarily an error) 2023-08-10T06:27:41,399 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/merge already deleted, retry=false 2023-08-10T06:27:41,415 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Unable to get data of znode /1/snapshot-cleanup because node does not exist (not necessarily an error) 2023-08-10T06:27:41,417 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/snapshot-cleanup already deleted, retry=false 2023-08-10T06:27:41,422 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-08-10T06:27:41,422 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45101-0x10060ea6e8b0001, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-08-10T06:27:41,422 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:27:41,422 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45101-0x10060ea6e8b0001, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:27:41,423 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(803): Active/primary master=jenkins-hbase3.apache.org,33705,1691648856595, sessionid=0x10060ea6e8b0000, setting cluster-up flag (Was=false) 2023-08-10T06:27:41,433 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2023-08-10T06:27:41,441 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45101-0x10060ea6e8b0001, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:27:41,441 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:27:41,447 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/flush-table-proc/acquired, /1/flush-table-proc/reached, /1/flush-table-proc/abort 2023-08-10T06:27:41,448 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,33705,1691648856595 2023-08-10T06:27:41,451 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:27:41,451 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45101-0x10060ea6e8b0001, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:27:41,458 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/rolllog-proc/acquired, /1/rolllog-proc/reached, /1/rolllog-proc/abort 2023-08-10T06:27:41,460 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,33705,1691648856595 2023-08-10T06:27:41,466 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:27:41,466 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45101-0x10060ea6e8b0001, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:27:41,472 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/online-snapshot/acquired, /1/online-snapshot/reached, /1/online-snapshot/abort 2023-08-10T06:27:41,474 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,33705,1691648856595 2023-08-10T06:27:41,591 DEBUG [RS:0;jenkins-hbase3:45101 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;jenkins-hbase3:45101 2023-08-10T06:27:41,593 INFO [RS:0;jenkins-hbase3:45101 {}] regionserver.HRegionServer(997): ClusterId : e031a179-98a7-4aac-acc3-d4298684a610 2023-08-10T06:27:41,595 INFO [RS:0;jenkins-hbase3:45101 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2023-08-10T06:27:41,599 DEBUG [RS:0;jenkins-hbase3:45101 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2023-08-10T06:27:41,607 DEBUG [RS:0;jenkins-hbase3:45101 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2023-08-10T06:27:41,607 DEBUG [RS:0;jenkins-hbase3:45101 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2023-08-10T06:27:41,612 DEBUG [RS:0;jenkins-hbase3:45101 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2023-08-10T06:27:41,613 DEBUG [RS:0;jenkins-hbase3:45101 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2023-08-10T06:27:41,617 DEBUG [RS:0;jenkins-hbase3:45101 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2023-08-10T06:27:41,619 DEBUG [RS:0;jenkins-hbase3:45101 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@558e925a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:27:41,627 DEBUG [RS:0;jenkins-hbase3:45101 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c5bab17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-08-10T06:27:41,638 INFO [RS:0;jenkins-hbase3:45101 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2023-08-10T06:27:41,638 INFO [RS:0;jenkins-hbase3:45101 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2023-08-10T06:27:41,639 DEBUG [RS:0;jenkins-hbase3:45101 {}] regionserver.HRegionServer(1079): About to register with Master. 2023-08-10T06:27:41,639 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1033): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2023-08-10T06:27:41,646 INFO [RS:0;jenkins-hbase3:45101 {}] regionserver.HRegionServer(3023): reportForDuty to master=jenkins-hbase3.apache.org,33705,1691648856595 with isa=jenkins-hbase3.apache.org/172.31.12.81:45101, startcode=1691648859755 2023-08-10T06:27:41,653 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(574): slop=0.2 2023-08-10T06:27:41,660 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(285): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2023-08-10T06:27:41,675 DEBUG [RS:0;jenkins-hbase3:45101 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-08-10T06:27:41,674 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: jenkins-hbase3.apache.org,33705,1691648856595 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2023-08-10T06:27:41,680 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-08-10T06:27:41,680 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-08-10T06:27:41,680 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-08-10T06:27:41,680 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-08-10T06:27:41,681 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/jenkins-hbase3:0, corePoolSize=10, maxPoolSize=10 2023-08-10T06:27:41,681 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:27:41,681 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-08-10T06:27:41,681 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:27:41,688 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1691648891688 2023-08-10T06:27:41,693 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2023-08-10T06:27:41,696 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2023-08-10T06:27:41,696 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2023-08-10T06:27:41,698 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2023-08-10T06:27:41,702 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:27:41,702 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-08-10T06:27:41,714 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2023-08-10T06:27:41,714 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2023-08-10T06:27:41,715 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2023-08-10T06:27:41,721 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2023-08-10T06:27:41,721 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2023-08-10T06:27:41,724 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-08-10T06:27:41,731 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2023-08-10T06:27:41,736 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2023-08-10T06:27:41,739 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2023-08-10T06:27:41,751 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ReadOnlyZKClient(139): Connect 0x20ed217b to 127.0.0.1:61704 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:27:41,753 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2023-08-10T06:27:41,753 INFO [PEWorker-1 {}] regionserver.HRegion(7075): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e 2023-08-10T06:27:41,823 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ff5c55f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:27:41,824 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-08-10T06:27:41,841 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2023-08-10T06:27:41,843 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2023-08-10T06:27:41,846 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1691648861845,5,FailOnTimeoutGroup] 2023-08-10T06:27:41,846 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:57953, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-08-10T06:27:41,850 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1691648861846,5,FailOnTimeoutGroup] 2023-08-10T06:27:41,850 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-08-10T06:27:41,850 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1634): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2023-08-10T06:27:41,853 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2023-08-10T06:27:41,853 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2023-08-10T06:27:41,870 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33705 {}] master.ServerManager(403): Registering regionserver=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:27:41,898 DEBUG [RS:0;jenkins-hbase3:45101 {}] regionserver.HRegionServer(1705): Config from master: hbase.rootdir=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e 2023-08-10T06:27:41,898 DEBUG [RS:0;jenkins-hbase3:45101 {}] regionserver.HRegionServer(1705): Config from master: fs.defaultFS=hdfs://localhost:45379 2023-08-10T06:27:41,899 DEBUG [RS:0;jenkins-hbase3:45101 {}] regionserver.HRegionServer(1705): Config from master: hbase.master.info.port=44311 2023-08-10T06:27:41,907 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2023-08-10T06:27:41,908 DEBUG [RS:0;jenkins-hbase3:45101 {}] zookeeper.ZKUtil(111): regionserver:45101-0x10060ea6e8b0001, quorum=127.0.0.1:61704, baseZNode=/1 Set watcher on existing znode=/1/rs/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:27:41,909 WARN [RS:0;jenkins-hbase3:45101 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-08-10T06:27:41,909 INFO [RS:0;jenkins-hbase3:45101 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-08-10T06:27:41,909 DEBUG [RS:0;jenkins-hbase3:45101 {}] regionserver.HRegionServer(2060): logDir=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:27:41,913 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [jenkins-hbase3.apache.org,45101,1691648859755] 2023-08-10T06:27:41,937 DEBUG [RS:0;jenkins-hbase3:45101 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2023-08-10T06:27:41,952 INFO [RS:0;jenkins-hbase3:45101 {}] regionserver.MetricsRegionServerWrapperImpl(165): Computing regionserver metrics every 5000 milliseconds 2023-08-10T06:27:41,978 INFO [RS:0;jenkins-hbase3:45101 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=782.4 M, globalMemStoreLimitLowMark=743.3 M, Offheap=false 2023-08-10T06:27:41,984 INFO [RS:0;jenkins-hbase3:45101 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2023-08-10T06:27:41,984 INFO [RS:0;jenkins-hbase3:45101 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2023-08-10T06:27:41,986 INFO [RS:0;jenkins-hbase3:45101 {}] regionserver.HRegionServer$CompactionChecker(1947): CompactionChecker runs every PT1S 2023-08-10T06:27:42,000 INFO [RS:0;jenkins-hbase3:45101 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2023-08-10T06:27:42,000 DEBUG [RS:0;jenkins-hbase3:45101 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:27:42,000 DEBUG [RS:0;jenkins-hbase3:45101 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:27:42,000 DEBUG [RS:0;jenkins-hbase3:45101 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:27:42,001 DEBUG [RS:0;jenkins-hbase3:45101 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:27:42,001 DEBUG [RS:0;jenkins-hbase3:45101 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:27:42,001 DEBUG [RS:0;jenkins-hbase3:45101 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-08-10T06:27:42,001 DEBUG [RS:0;jenkins-hbase3:45101 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:27:42,001 DEBUG [RS:0;jenkins-hbase3:45101 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:27:42,001 DEBUG [RS:0;jenkins-hbase3:45101 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:27:42,002 DEBUG [RS:0;jenkins-hbase3:45101 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:27:42,002 DEBUG [RS:0;jenkins-hbase3:45101 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:27:42,002 DEBUG [RS:0;jenkins-hbase3:45101 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-08-10T06:27:42,006 INFO [RS:0;jenkins-hbase3:45101 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2023-08-10T06:27:42,006 INFO [RS:0;jenkins-hbase3:45101 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2023-08-10T06:27:42,006 INFO [RS:0;jenkins-hbase3:45101 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2023-08-10T06:27:42,006 INFO [RS:0;jenkins-hbase3:45101 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2023-08-10T06:27:42,006 INFO [RS:0;jenkins-hbase3:45101 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,45101,1691648859755-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-08-10T06:27:42,031 INFO [RS:0;jenkins-hbase3:45101 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2023-08-10T06:27:42,035 INFO [RS:0;jenkins-hbase3:45101 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,45101,1691648859755-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2023-08-10T06:27:42,060 INFO [RS:0;jenkins-hbase3:45101 {}] regionserver.Replication(204): jenkins-hbase3.apache.org,45101,1691648859755 started 2023-08-10T06:27:42,061 INFO [RS:0;jenkins-hbase3:45101 {}] regionserver.HRegionServer(1747): Serving as jenkins-hbase3.apache.org,45101,1691648859755, RpcServer on jenkins-hbase3.apache.org/172.31.12.81:45101, sessionid=0x10060ea6e8b0001 2023-08-10T06:27:42,061 DEBUG [RS:0;jenkins-hbase3:45101 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2023-08-10T06:27:42,062 DEBUG [RS:0;jenkins-hbase3:45101 {}] flush.RegionServerFlushTableProcedureManager(106): Start region server flush procedure manager jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:27:42,062 DEBUG [RS:0;jenkins-hbase3:45101 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,45101,1691648859755' 2023-08-10T06:27:42,062 DEBUG [RS:0;jenkins-hbase3:45101 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/flush-table-proc/abort' 2023-08-10T06:27:42,063 DEBUG [RS:0;jenkins-hbase3:45101 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/flush-table-proc/acquired' 2023-08-10T06:27:42,064 DEBUG [RS:0;jenkins-hbase3:45101 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2023-08-10T06:27:42,064 DEBUG [RS:0;jenkins-hbase3:45101 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2023-08-10T06:27:42,064 DEBUG [RS:0;jenkins-hbase3:45101 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,45101,1691648859755' 2023-08-10T06:27:42,064 DEBUG [RS:0;jenkins-hbase3:45101 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:27:42,065 DEBUG [RS:0;jenkins-hbase3:45101 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:27:42,065 INFO [RS:0;jenkins-hbase3:45101 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2023-08-10T06:27:42,066 DEBUG [RS:0;jenkins-hbase3:45101 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2023-08-10T06:27:42,066 DEBUG [RS:0;jenkins-hbase3:45101 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2023-08-10T06:27:42,066 DEBUG [RS:0;jenkins-hbase3:45101 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:27:42,066 DEBUG [RS:0;jenkins-hbase3:45101 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,45101,1691648859755' 2023-08-10T06:27:42,066 DEBUG [RS:0;jenkins-hbase3:45101 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/online-snapshot/abort' 2023-08-10T06:27:42,067 DEBUG [RS:0;jenkins-hbase3:45101 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/online-snapshot/acquired' 2023-08-10T06:27:42,067 DEBUG [RS:0;jenkins-hbase3:45101 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2023-08-10T06:27:42,067 INFO [RS:0;jenkins-hbase3:45101 {}] quotas.RegionServerRpcQuotaManager(63): Quota support disabled 2023-08-10T06:27:42,068 INFO [RS:0;jenkins-hbase3:45101 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2023-08-10T06:27:42,181 INFO [RS:0;jenkins-hbase3:45101 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-08-10T06:27:42,185 INFO [RS:0;jenkins-hbase3:45101 {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C45101%2C1691648859755, suffix=, logDir=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755, archiveDir=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/oldWALs, maxLogs=32 2023-08-10T06:27:42,201 DEBUG [RS:0;jenkins-hbase3:45101 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691648862187, exclude list is [], retry=0 2023-08-10T06:27:42,208 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK] 2023-08-10T06:27:42,213 INFO [RS:0;jenkins-hbase3:45101 {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691648862187 2023-08-10T06:27:42,214 DEBUG [RS:0;jenkins-hbase3:45101 {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK]] 2023-08-10T06:27:42,247 DEBUG [PEWorker-1 {}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:27:42,250 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-08-10T06:27:42,252 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-08-10T06:27:42,252 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:27:42,253 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-08-10T06:27:42,253 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-08-10T06:27:42,255 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-08-10T06:27:42,255 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:27:42,256 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-08-10T06:27:42,256 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-08-10T06:27:42,258 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-08-10T06:27:42,258 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:27:42,259 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-08-10T06:27:42,260 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/hbase/meta/1588230740 2023-08-10T06:27:42,261 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/hbase/meta/1588230740 2023-08-10T06:27:42,263 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-08-10T06:27:42,265 DEBUG [PEWorker-1 {}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-08-10T06:27:42,268 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:27:42,269 INFO [PEWorker-1 {}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10706326560, jitterRate=-0.002895638346672058}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-08-10T06:27:42,271 DEBUG [PEWorker-1 {}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-08-10T06:27:42,271 DEBUG [PEWorker-1 {}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-08-10T06:27:42,271 INFO [PEWorker-1 {}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-08-10T06:27:42,271 DEBUG [PEWorker-1 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-08-10T06:27:42,271 DEBUG [PEWorker-1 {}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-08-10T06:27:42,271 DEBUG [PEWorker-1 {}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-08-10T06:27:42,273 INFO [PEWorker-1 {}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-08-10T06:27:42,273 DEBUG [PEWorker-1 {}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-08-10T06:27:42,277 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2023-08-10T06:27:42,277 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2023-08-10T06:27:42,289 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2023-08-10T06:27:42,301 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2023-08-10T06:27:42,306 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2023-08-10T06:27:42,458 DEBUG [jenkins-hbase3:33705 {}] assignment.AssignmentManager(2303): Processing assignQueue; systemServersCount=1, allServersCount=1 2023-08-10T06:27:42,467 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:27:42,473 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,45101,1691648859755, state=OPENING 2023-08-10T06:27:42,482 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2023-08-10T06:27:42,487 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45101-0x10060ea6e8b0001, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:27:42,487 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:27:42,487 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-08-10T06:27:42,487 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-08-10T06:27:42,492 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,45101,1691648859755}] 2023-08-10T06:27:42,688 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(721): New admin connection to jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:27:42,691 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=AdminService, sasl=false 2023-08-10T06:27:42,695 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:38738, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2023-08-10T06:27:42,710 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(130): Open hbase:meta,,1.1588230740 2023-08-10T06:27:42,711 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-08-10T06:27:42,712 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2023-08-10T06:27:42,716 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta, suffix=.meta, logDir=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755, archiveDir=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/oldWALs, maxLogs=32 2023-08-10T06:27:42,730 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691648862717.meta, exclude list is [], retry=0 2023-08-10T06:27:42,736 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK] 2023-08-10T06:27:42,740 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691648862717.meta 2023-08-10T06:27:42,741 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK]] 2023-08-10T06:27:42,741 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7254): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:27:42,747 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:27:42,748 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2023-08-10T06:27:42,770 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7955): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2023-08-10T06:27:42,770 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(393): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2023-08-10T06:27:42,775 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2023-08-10T06:27:42,776 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:27:42,776 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7296): checking encryption for 1588230740 2023-08-10T06:27:42,776 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7299): checking classloading for 1588230740 2023-08-10T06:27:42,779 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-08-10T06:27:42,780 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-08-10T06:27:42,780 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:27:42,781 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-08-10T06:27:42,781 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-08-10T06:27:42,783 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-08-10T06:27:42,783 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:27:42,783 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-08-10T06:27:42,784 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-08-10T06:27:42,785 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-08-10T06:27:42,785 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:27:42,786 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-08-10T06:27:42,787 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/hbase/meta/1588230740 2023-08-10T06:27:42,790 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/hbase/meta/1588230740 2023-08-10T06:27:42,792 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-08-10T06:27:42,794 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-08-10T06:27:42,796 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11378033920, jitterRate=0.05966198444366455}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-08-10T06:27:42,797 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-08-10T06:27:42,809 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2552): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1691648862677 2023-08-10T06:27:42,828 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2579): Finished post open deploy task for hbase:meta,,1.1588230740 2023-08-10T06:27:42,828 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(158): Opened hbase:meta,,1.1588230740 2023-08-10T06:27:42,830 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:27:42,831 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,45101,1691648859755, state=OPEN 2023-08-10T06:27:42,834 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45101-0x10060ea6e8b0001, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-08-10T06:27:42,834 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-08-10T06:27:42,834 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-08-10T06:27:42,834 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-08-10T06:27:42,838 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=3, resume processing ppid=2 2023-08-10T06:27:42,838 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,45101,1691648859755 in 342 msec 2023-08-10T06:27:42,847 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=2, resume processing ppid=1 2023-08-10T06:27:42,847 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 551 msec 2023-08-10T06:27:42,852 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 1.3120 sec 2023-08-10T06:27:42,852 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1062): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1691648862852, completionTime=-1 2023-08-10T06:27:42,852 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ServerManager(830): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2023-08-10T06:27:42,852 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1629): Joining cluster... 2023-08-10T06:27:43,061 DEBUG [hconnection-0x69012128-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:27:43,065 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:38742, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:27:43,083 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1641): Number of RegionServers=1 2023-08-10T06:27:43,083 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1691648923083 2023-08-10T06:27:43,083 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1691648983083 2023-08-10T06:27:43,083 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1648): Joined the cluster in 230 msec 2023-08-10T06:27:43,114 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,33705,1691648856595-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2023-08-10T06:27:43,114 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,33705,1691648856595-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2023-08-10T06:27:43,115 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,33705,1691648856595-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2023-08-10T06:27:43,117 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-jenkins-hbase3:33705, period=300000, unit=MILLISECONDS is enabled. 2023-08-10T06:27:43,117 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2023-08-10T06:27:43,124 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] janitor.CatalogJanitor(175): 2023-08-10T06:27:43,139 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2023-08-10T06:27:43,140 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2367): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-08-10T06:27:43,150 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1033): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2023-08-10T06:27:43,154 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2023-08-10T06:27:43,159 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:27:43,160 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-08-10T06:27:43,581 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 7ccd561208a3736e53c95b12322d6daf, NAME => 'hbase:namespace,,1691648863140.7ccd561208a3736e53c95b12322d6daf.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e 2023-08-10T06:27:43,593 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:namespace,,1691648863140.7ccd561208a3736e53c95b12322d6daf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:27:43,593 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1668): Closing 7ccd561208a3736e53c95b12322d6daf, disabling compactions & flushes 2023-08-10T06:27:43,593 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:namespace,,1691648863140.7ccd561208a3736e53c95b12322d6daf. 2023-08-10T06:27:43,594 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1691648863140.7ccd561208a3736e53c95b12322d6daf. 2023-08-10T06:27:43,594 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1691648863140.7ccd561208a3736e53c95b12322d6daf. after waiting 0 ms 2023-08-10T06:27:43,594 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1691648863140.7ccd561208a3736e53c95b12322d6daf. 2023-08-10T06:27:43,594 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1909): Closed hbase:namespace,,1691648863140.7ccd561208a3736e53c95b12322d6daf. 2023-08-10T06:27:43,594 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1622): Region close journal for 7ccd561208a3736e53c95b12322d6daf: 2023-08-10T06:27:43,596 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2023-08-10T06:27:43,610 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1691648863140.7ccd561208a3736e53c95b12322d6daf.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1691648863598"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691648863598"}]},"ts":"1691648863598"} 2023-08-10T06:27:43,644 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:27:43,646 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-08-10T06:27:43,651 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648863646"}]},"ts":"1691648863646"} 2023-08-10T06:27:43,659 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2023-08-10T06:27:43,664 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=7ccd561208a3736e53c95b12322d6daf, ASSIGN}] 2023-08-10T06:27:43,666 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=7ccd561208a3736e53c95b12322d6daf, ASSIGN 2023-08-10T06:27:43,668 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=7ccd561208a3736e53c95b12322d6daf, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45101,1691648859755; forceNewPlan=false, retain=false 2023-08-10T06:27:43,819 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=7ccd561208a3736e53c95b12322d6daf, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:27:43,823 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 7ccd561208a3736e53c95b12322d6daf, server=jenkins-hbase3.apache.org,45101,1691648859755}] 2023-08-10T06:27:43,983 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(130): Open hbase:namespace,,1691648863140.7ccd561208a3736e53c95b12322d6daf. 2023-08-10T06:27:43,984 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7254): Opening region: {ENCODED => 7ccd561208a3736e53c95b12322d6daf, NAME => 'hbase:namespace,,1691648863140.7ccd561208a3736e53c95b12322d6daf.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:27:43,985 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:27:43,985 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 7ccd561208a3736e53c95b12322d6daf 2023-08-10T06:27:43,986 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(888): Instantiated hbase:namespace,,1691648863140.7ccd561208a3736e53c95b12322d6daf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:27:43,986 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7296): checking encryption for 7ccd561208a3736e53c95b12322d6daf 2023-08-10T06:27:43,986 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7299): checking classloading for 7ccd561208a3736e53c95b12322d6daf 2023-08-10T06:27:43,988 INFO [StoreOpener-7ccd561208a3736e53c95b12322d6daf-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 7ccd561208a3736e53c95b12322d6daf 2023-08-10T06:27:43,990 INFO [StoreOpener-7ccd561208a3736e53c95b12322d6daf-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7ccd561208a3736e53c95b12322d6daf columnFamilyName info 2023-08-10T06:27:43,990 DEBUG [StoreOpener-7ccd561208a3736e53c95b12322d6daf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:27:43,991 INFO [StoreOpener-7ccd561208a3736e53c95b12322d6daf-1 {}] regionserver.HStore(324): Store=7ccd561208a3736e53c95b12322d6daf/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:27:43,992 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/hbase/namespace/7ccd561208a3736e53c95b12322d6daf 2023-08-10T06:27:43,993 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/hbase/namespace/7ccd561208a3736e53c95b12322d6daf 2023-08-10T06:27:43,996 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1076): writing seq id for 7ccd561208a3736e53c95b12322d6daf 2023-08-10T06:27:43,999 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/hbase/namespace/7ccd561208a3736e53c95b12322d6daf/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:27:44,000 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1093): Opened 7ccd561208a3736e53c95b12322d6daf; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11452850400, jitterRate=0.0666298121213913}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-08-10T06:27:44,001 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(992): Region open journal for 7ccd561208a3736e53c95b12322d6daf: 2023-08-10T06:27:44,003 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2552): Post open deploy tasks for hbase:namespace,,1691648863140.7ccd561208a3736e53c95b12322d6daf., pid=6, masterSystemTime=1691648863977 2023-08-10T06:27:44,006 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2579): Finished post open deploy task for hbase:namespace,,1691648863140.7ccd561208a3736e53c95b12322d6daf. 2023-08-10T06:27:44,006 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(158): Opened hbase:namespace,,1691648863140.7ccd561208a3736e53c95b12322d6daf. 2023-08-10T06:27:44,007 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=7ccd561208a3736e53c95b12322d6daf, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:27:44,015 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=6, resume processing ppid=5 2023-08-10T06:27:44,015 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 7ccd561208a3736e53c95b12322d6daf, server=jenkins-hbase3.apache.org,45101,1691648859755 in 188 msec 2023-08-10T06:27:44,018 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=5, resume processing ppid=4 2023-08-10T06:27:44,018 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=7ccd561208a3736e53c95b12322d6daf, ASSIGN in 351 msec 2023-08-10T06:27:44,019 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-08-10T06:27:44,019 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648864019"}]},"ts":"1691648864019"} 2023-08-10T06:27:44,022 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2023-08-10T06:27:44,028 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2023-08-10T06:27:44,031 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 887 msec 2023-08-10T06:27:44,054 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Set watcher on znode that does not yet exist, /1/namespace 2023-08-10T06:27:44,056 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/namespace 2023-08-10T06:27:44,056 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:27:44,056 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45101-0x10060ea6e8b0001, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:27:44,103 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1033): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2023-08-10T06:27:44,121 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-08-10T06:27:44,127 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 31 msec 2023-08-10T06:27:44,138 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1033): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2023-08-10T06:27:44,152 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-08-10T06:27:44,156 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 17 msec 2023-08-10T06:27:44,165 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/default 2023-08-10T06:27:44,169 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/hbase 2023-08-10T06:27:44,169 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1192): Master has completed initialization 4.223sec 2023-08-10T06:27:44,171 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2023-08-10T06:27:44,173 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2023-08-10T06:27:44,174 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2023-08-10T06:27:44,175 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2023-08-10T06:27:44,175 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2023-08-10T06:27:44,177 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,33705,1691648856595-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-08-10T06:27:44,177 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,33705,1691648856595-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2023-08-10T06:27:44,191 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1295): Balancer post startup initialization complete, took 0 seconds 2023-08-10T06:27:44,193 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2023-08-10T06:27:44,197 DEBUG [Listener at localhost/36815 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x083ba722 to 127.0.0.1:61704 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:27:44,202 DEBUG [Listener at localhost/36815 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e1b854, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:27:44,207 DEBUG [Listener at localhost/36815 {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2023-08-10T06:27:44,207 DEBUG [Listener at localhost/36815 {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2023-08-10T06:27:44,223 DEBUG [hconnection-0x65702ead-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:27:44,236 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:38750, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:27:44,247 INFO [Listener at localhost/36815 {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=jenkins-hbase3.apache.org,33705,1691648856595 2023-08-10T06:27:44,247 INFO [Listener at localhost/36815 {}] hbase.HBaseTestingUtility(2786): Starting mini mapreduce cluster... 2023-08-10T06:27:44,248 INFO [Listener at localhost/36815 {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/test.cache.data in system properties and HBase conf 2023-08-10T06:27:44,248 INFO [Listener at localhost/36815 {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/hadoop.tmp.dir in system properties and HBase conf 2023-08-10T06:27:44,248 INFO [Listener at localhost/36815 {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/hadoop.log.dir in system properties and HBase conf 2023-08-10T06:27:44,248 INFO [Listener at localhost/36815 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir in system properties and HBase conf 2023-08-10T06:27:44,248 INFO [Listener at localhost/36815 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-08-10T06:27:44,248 INFO [Listener at localhost/36815 {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-08-10T06:27:44,248 INFO [Listener at localhost/36815 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-08-10T06:27:44,249 INFO [Listener at localhost/36815 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-08-10T06:27:44,249 INFO [Listener at localhost/36815 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-08-10T06:27:44,249 INFO [Listener at localhost/36815 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-08-10T06:27:44,249 INFO [Listener at localhost/36815 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-08-10T06:27:44,249 INFO [Listener at localhost/36815 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-08-10T06:27:44,249 INFO [Listener at localhost/36815 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-08-10T06:27:44,249 INFO [Listener at localhost/36815 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/dfs.journalnode.edits.dir in system properties and HBase conf 2023-08-10T06:27:44,249 INFO [Listener at localhost/36815 {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-08-10T06:27:44,249 INFO [Listener at localhost/36815 {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/nfs.dump.dir in system properties and HBase conf 2023-08-10T06:27:44,249 INFO [Listener at localhost/36815 {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/java.io.tmpdir in system properties and HBase conf 2023-08-10T06:27:44,249 INFO [Listener at localhost/36815 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/dfs.journalnode.edits.dir in system properties and HBase conf 2023-08-10T06:27:44,249 INFO [Listener at localhost/36815 {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-08-10T06:27:44,249 INFO [Listener at localhost/36815 {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2023-08-10T06:27:45,750 WARN [Thread-249 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-08-10T06:27:46,128 INFO [Thread-249 {}] log.Slf4jLog(67): jetty-6.1.26 2023-08-10T06:27:46,134 INFO [Thread-249 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/jobhistory to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_35243_jobhistory____wxb5p5/webapp 2023-08-10T06:27:46,311 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices as a root resource class 2023-08-10T06:27:46,311 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver as a provider class 2023-08-10T06:27:46,311 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-08-10T06:27:46,314 INFO [Thread-249 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-08-10T06:27:46,398 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:27:46,514 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:27:46,763 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2023-08-10T06:27:46,763 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2023-08-10T06:27:46,772 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2023-08-10T06:27:46,772 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2023-08-10T06:27:46,775 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:27:46,775 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver Metrics about HBase RegionObservers 2023-08-10T06:27:46,776 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-08-10T06:27:46,776 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2023-08-10T06:27:46,776 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2023-08-10T06:27:46,776 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2023-08-10T06:27:46,814 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices to GuiceManagedComponentProvider with the scope "PerRequest" 2023-08-10T06:27:46,851 INFO [Thread-249 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:35243 2023-08-10T06:27:47,244 WARN [Listener at jenkins-hbase3.apache.org/44395 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-08-10T06:27:47,325 WARN [Listener at jenkins-hbase3.apache.org/44395 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-08-10T06:27:47,325 INFO [Listener at jenkins-hbase3.apache.org/44395 {}] log.Slf4jLog(67): jetty-6.1.26 2023-08-10T06:27:47,332 INFO [Listener at jenkins-hbase3.apache.org/44395 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/cluster to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_46845_cluster____tcltxo/webapp 2023-08-10T06:27:47,407 INFO [Listener at jenkins-hbase3.apache.org/44395 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver as a provider class 2023-08-10T06:27:47,407 INFO [Listener at jenkins-hbase3.apache.org/44395 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices as a root resource class 2023-08-10T06:27:47,407 INFO [Listener at jenkins-hbase3.apache.org/44395 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-08-10T06:27:47,407 INFO [Listener at jenkins-hbase3.apache.org/44395 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-08-10T06:27:47,417 INFO [Listener at jenkins-hbase3.apache.org/44395 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:27:47,624 INFO [Listener at jenkins-hbase3.apache.org/44395 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:27:47,706 INFO [Listener at jenkins-hbase3.apache.org/44395 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:27:47,713 INFO [Listener at jenkins-hbase3.apache.org/44395 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:46845 2023-08-10T06:27:48,075 WARN [Listener at jenkins-hbase3.apache.org/33647 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-08-10T06:27:48,103 WARN [Listener at jenkins-hbase3.apache.org/33647 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-08-10T06:27:48,104 INFO [Listener at jenkins-hbase3.apache.org/33647 {}] log.Slf4jLog(67): jetty-6.1.26 2023-08-10T06:27:48,111 INFO [Listener at jenkins-hbase3.apache.org/33647 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_33521_node____eo2j1v/webapp 2023-08-10T06:27:48,163 INFO [Listener at jenkins-hbase3.apache.org/33647 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-08-10T06:27:48,163 INFO [Listener at jenkins-hbase3.apache.org/33647 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-08-10T06:27:48,163 INFO [Listener at jenkins-hbase3.apache.org/33647 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-08-10T06:27:48,164 INFO [Listener at jenkins-hbase3.apache.org/33647 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-08-10T06:27:48,171 INFO [Listener at jenkins-hbase3.apache.org/33647 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:27:48,193 INFO [Listener at jenkins-hbase3.apache.org/33647 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:27:48,277 INFO [Listener at jenkins-hbase3.apache.org/33647 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:27:48,286 INFO [Listener at jenkins-hbase3.apache.org/33647 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:33521 2023-08-10T06:27:48,327 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-08-10T06:27:48,416 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2023-08-10T06:27:48,419 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2023-08-10T06:27:48,497 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-08-10T06:27:48,506 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-08-10T06:27:48,507 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] log.Slf4jLog(67): jetty-6.1.26 2023-08-10T06:27:48,516 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_42293_node____eaw3hp/webapp 2023-08-10T06:27:48,573 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-08-10T06:27:48,573 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-08-10T06:27:48,574 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-08-10T06:27:48,574 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-08-10T06:27:48,585 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:27:48,600 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:27:48,685 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:27:48,699 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:42293 2023-08-10T06:27:48,734 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.HBaseTestingUtility(2821): Mini mapreduce cluster started 2023-08-10T06:27:48,734 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] backup.TestBackupBase(301): ROOTDIR hdfs://localhost:45379/backupUT 2023-08-10T06:27:48,746 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:27:48,750 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:56314, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:27:48,760 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'ns1'} 2023-08-10T06:27:48,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=9, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns1 2023-08-10T06:27:48,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=9 2023-08-10T06:27:48,788 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-08-10T06:27:48,794 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=9, state=SUCCESS; CreateNamespaceProcedure, namespace=ns1 in 31 msec 2023-08-10T06:27:48,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=9 2023-08-10T06:27:48,884 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'ns2'} 2023-08-10T06:27:48,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=10, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns2 2023-08-10T06:27:48,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=10 2023-08-10T06:27:48,903 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-08-10T06:27:48,910 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=10, state=SUCCESS; CreateNamespaceProcedure, namespace=ns2 in 24 msec 2023-08-10T06:27:48,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=10 2023-08-10T06:27:48,996 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'ns3'} 2023-08-10T06:27:48,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=11, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns3 2023-08-10T06:27:49,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=11 2023-08-10T06:27:49,018 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-08-10T06:27:49,023 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=11, state=SUCCESS; CreateNamespaceProcedure, namespace=ns3 in 25 msec 2023-08-10T06:27:49,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=11 2023-08-10T06:27:49,109 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'ns4'} 2023-08-10T06:27:49,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=12, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns4 2023-08-10T06:27:49,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=12 2023-08-10T06:27:49,124 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-08-10T06:27:49,129 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=12, state=SUCCESS; CreateNamespaceProcedure, namespace=ns4 in 18 msec 2023-08-10T06:27:49,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=12 2023-08-10T06:27:49,232 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'test-1691648868734', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:27:49,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=test-1691648868734 2023-08-10T06:27:49,236 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=test-1691648868734 execute state=CREATE_TABLE_PRE_OPERATION 2023-08-10T06:27:49,237 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:27:49,238 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=test-1691648868734 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-08-10T06:27:49,243 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "test-1691648868734" procId is: 13 2023-08-10T06:27:49,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-08-10T06:27:49,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-08-10T06:27:49,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-08-10T06:27:49,668 INFO [RegionOpenAndInit-test-1691648868734-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => f320d55a87f9e601302313b6d2cd1a75, NAME => 'test-1691648868734,,1691648869226.f320d55a87f9e601302313b6d2cd1a75.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1691648868734', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e 2023-08-10T06:27:49,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-08-10T06:27:50,088 DEBUG [RegionOpenAndInit-test-1691648868734-pool-0 {}] regionserver.HRegion(888): Instantiated test-1691648868734,,1691648869226.f320d55a87f9e601302313b6d2cd1a75.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:27:50,089 DEBUG [RegionOpenAndInit-test-1691648868734-pool-0 {}] regionserver.HRegion(1668): Closing f320d55a87f9e601302313b6d2cd1a75, disabling compactions & flushes 2023-08-10T06:27:50,089 INFO [RegionOpenAndInit-test-1691648868734-pool-0 {}] regionserver.HRegion(1690): Closing region test-1691648868734,,1691648869226.f320d55a87f9e601302313b6d2cd1a75. 2023-08-10T06:27:50,089 DEBUG [RegionOpenAndInit-test-1691648868734-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1691648868734,,1691648869226.f320d55a87f9e601302313b6d2cd1a75. 2023-08-10T06:27:50,089 DEBUG [RegionOpenAndInit-test-1691648868734-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on test-1691648868734,,1691648869226.f320d55a87f9e601302313b6d2cd1a75. after waiting 0 ms 2023-08-10T06:27:50,089 DEBUG [RegionOpenAndInit-test-1691648868734-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region test-1691648868734,,1691648869226.f320d55a87f9e601302313b6d2cd1a75. 2023-08-10T06:27:50,089 INFO [RegionOpenAndInit-test-1691648868734-pool-0 {}] regionserver.HRegion(1909): Closed test-1691648868734,,1691648869226.f320d55a87f9e601302313b6d2cd1a75. 2023-08-10T06:27:50,089 DEBUG [RegionOpenAndInit-test-1691648868734-pool-0 {}] regionserver.HRegion(1622): Region close journal for f320d55a87f9e601302313b6d2cd1a75: 2023-08-10T06:27:50,091 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=test-1691648868734 execute state=CREATE_TABLE_ADD_TO_META 2023-08-10T06:27:50,091 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"test-1691648868734,,1691648869226.f320d55a87f9e601302313b6d2cd1a75.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1691648870091"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691648870091"}]},"ts":"1691648870091"} 2023-08-10T06:27:50,093 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:27:50,095 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=test-1691648868734 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-08-10T06:27:50,095 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1691648868734","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648870095"}]},"ts":"1691648870095"} 2023-08-10T06:27:50,097 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1691648868734, state=ENABLING in hbase:meta 2023-08-10T06:27:50,103 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1691648868734, region=f320d55a87f9e601302313b6d2cd1a75, ASSIGN}] 2023-08-10T06:27:50,105 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1691648868734, region=f320d55a87f9e601302313b6d2cd1a75, ASSIGN 2023-08-10T06:27:50,106 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=test-1691648868734, region=f320d55a87f9e601302313b6d2cd1a75, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45101,1691648859755; forceNewPlan=false, retain=false 2023-08-10T06:27:50,257 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=f320d55a87f9e601302313b6d2cd1a75, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:27:50,260 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE; OpenRegionProcedure f320d55a87f9e601302313b6d2cd1a75, server=jenkins-hbase3.apache.org,45101,1691648859755}] 2023-08-10T06:27:50,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-08-10T06:27:50,422 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(130): Open test-1691648868734,,1691648869226.f320d55a87f9e601302313b6d2cd1a75. 2023-08-10T06:27:50,422 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7254): Opening region: {ENCODED => f320d55a87f9e601302313b6d2cd1a75, NAME => 'test-1691648868734,,1691648869226.f320d55a87f9e601302313b6d2cd1a75.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:27:50,422 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:27:50,422 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1691648868734 f320d55a87f9e601302313b6d2cd1a75 2023-08-10T06:27:50,422 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(888): Instantiated test-1691648868734,,1691648869226.f320d55a87f9e601302313b6d2cd1a75.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:27:50,422 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7296): checking encryption for f320d55a87f9e601302313b6d2cd1a75 2023-08-10T06:27:50,423 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7299): checking classloading for f320d55a87f9e601302313b6d2cd1a75 2023-08-10T06:27:50,425 INFO [StoreOpener-f320d55a87f9e601302313b6d2cd1a75-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region f320d55a87f9e601302313b6d2cd1a75 2023-08-10T06:27:50,427 INFO [StoreOpener-f320d55a87f9e601302313b6d2cd1a75-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f320d55a87f9e601302313b6d2cd1a75 columnFamilyName f 2023-08-10T06:27:50,427 DEBUG [StoreOpener-f320d55a87f9e601302313b6d2cd1a75-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:27:50,428 INFO [StoreOpener-f320d55a87f9e601302313b6d2cd1a75-1 {}] regionserver.HStore(324): Store=f320d55a87f9e601302313b6d2cd1a75/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:27:50,429 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/f320d55a87f9e601302313b6d2cd1a75 2023-08-10T06:27:50,430 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/f320d55a87f9e601302313b6d2cd1a75 2023-08-10T06:27:50,434 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1076): writing seq id for f320d55a87f9e601302313b6d2cd1a75 2023-08-10T06:27:50,439 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/f320d55a87f9e601302313b6d2cd1a75/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:27:50,440 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): Opened f320d55a87f9e601302313b6d2cd1a75; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10505275680, jitterRate=-0.02161996066570282}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-08-10T06:27:50,441 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(992): Region open journal for f320d55a87f9e601302313b6d2cd1a75: 2023-08-10T06:27:50,442 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2552): Post open deploy tasks for test-1691648868734,,1691648869226.f320d55a87f9e601302313b6d2cd1a75., pid=15, masterSystemTime=1691648870414 2023-08-10T06:27:50,445 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2579): Finished post open deploy task for test-1691648868734,,1691648869226.f320d55a87f9e601302313b6d2cd1a75. 2023-08-10T06:27:50,445 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(158): Opened test-1691648868734,,1691648869226.f320d55a87f9e601302313b6d2cd1a75. 2023-08-10T06:27:50,446 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=f320d55a87f9e601302313b6d2cd1a75, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:27:50,450 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=15, resume processing ppid=14 2023-08-10T06:27:50,451 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=15, ppid=14, state=SUCCESS; OpenRegionProcedure f320d55a87f9e601302313b6d2cd1a75, server=jenkins-hbase3.apache.org,45101,1691648859755 in 188 msec 2023-08-10T06:27:50,453 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=14, resume processing ppid=13 2023-08-10T06:27:50,453 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=14, ppid=13, state=SUCCESS; TransitRegionStateProcedure table=test-1691648868734, region=f320d55a87f9e601302313b6d2cd1a75, ASSIGN in 348 msec 2023-08-10T06:27:50,454 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=test-1691648868734 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-08-10T06:27:50,455 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1691648868734","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648870454"}]},"ts":"1691648870454"} 2023-08-10T06:27:50,456 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1691648868734, state=ENABLED in hbase:meta 2023-08-10T06:27:50,459 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=test-1691648868734 execute state=CREATE_TABLE_POST_OPERATION 2023-08-10T06:27:50,461 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=13, state=SUCCESS; CreateTableProcedure table=test-1691648868734 in 1.2270 sec 2023-08-10T06:27:51,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-08-10T06:27:51,359 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: default:test-1691648868734, procId: 13 completed 2023-08-10T06:27:51,360 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x21f31315 to 127.0.0.1:61704 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:27:51,366 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@649eec50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:27:51,368 DEBUG [hconnection-0x1e10b6ae-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:27:51,371 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:44738, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:27:51,376 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HRegion(8210): writing data to region test-1691648868734,,1691648869226.f320d55a87f9e601302313b6d2cd1a75. with WAL disabled. Data may be lost in the event of a crash. 2023-08-10T06:27:51,483 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'ns2:test-16916488687341', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:27:51,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns2:test-16916488687341 2023-08-10T06:27:51,487 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16916488687341 execute state=CREATE_TABLE_PRE_OPERATION 2023-08-10T06:27:51,487 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:27:51,487 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns2" qualifier: "test-16916488687341" procId is: 16 2023-08-10T06:27:51,488 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns2:test-16916488687341 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-08-10T06:27:51,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-08-10T06:27:51,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-08-10T06:27:51,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-08-10T06:27:51,905 INFO [RegionOpenAndInit-ns2:test-16916488687341-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 455351a9679f34a345fb40a29340fd37, NAME => 'ns2:test-16916488687341,,1691648871482.455351a9679f34a345fb40a29340fd37.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-16916488687341', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e 2023-08-10T06:27:52,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-08-10T06:27:52,319 DEBUG [RegionOpenAndInit-ns2:test-16916488687341-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:test-16916488687341,,1691648871482.455351a9679f34a345fb40a29340fd37.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:27:52,319 DEBUG [RegionOpenAndInit-ns2:test-16916488687341-pool-0 {}] regionserver.HRegion(1668): Closing 455351a9679f34a345fb40a29340fd37, disabling compactions & flushes 2023-08-10T06:27:52,319 INFO [RegionOpenAndInit-ns2:test-16916488687341-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:test-16916488687341,,1691648871482.455351a9679f34a345fb40a29340fd37. 2023-08-10T06:27:52,319 DEBUG [RegionOpenAndInit-ns2:test-16916488687341-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16916488687341,,1691648871482.455351a9679f34a345fb40a29340fd37. 2023-08-10T06:27:52,319 DEBUG [RegionOpenAndInit-ns2:test-16916488687341-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16916488687341,,1691648871482.455351a9679f34a345fb40a29340fd37. after waiting 0 ms 2023-08-10T06:27:52,319 DEBUG [RegionOpenAndInit-ns2:test-16916488687341-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16916488687341,,1691648871482.455351a9679f34a345fb40a29340fd37. 2023-08-10T06:27:52,319 INFO [RegionOpenAndInit-ns2:test-16916488687341-pool-0 {}] regionserver.HRegion(1909): Closed ns2:test-16916488687341,,1691648871482.455351a9679f34a345fb40a29340fd37. 2023-08-10T06:27:52,319 DEBUG [RegionOpenAndInit-ns2:test-16916488687341-pool-0 {}] regionserver.HRegion(1622): Region close journal for 455351a9679f34a345fb40a29340fd37: 2023-08-10T06:27:52,320 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns2:test-16916488687341 execute state=CREATE_TABLE_ADD_TO_META 2023-08-10T06:27:52,321 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:test-16916488687341,,1691648871482.455351a9679f34a345fb40a29340fd37.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1691648872321"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691648872321"}]},"ts":"1691648872321"} 2023-08-10T06:27:52,323 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:27:52,325 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns2:test-16916488687341 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-08-10T06:27:52,325 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16916488687341","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648872325"}]},"ts":"1691648872325"} 2023-08-10T06:27:52,326 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16916488687341, state=ENABLING in hbase:meta 2023-08-10T06:27:52,331 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16916488687341, region=455351a9679f34a345fb40a29340fd37, ASSIGN}] 2023-08-10T06:27:52,333 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16916488687341, region=455351a9679f34a345fb40a29340fd37, ASSIGN 2023-08-10T06:27:52,334 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:test-16916488687341, region=455351a9679f34a345fb40a29340fd37, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45101,1691648859755; forceNewPlan=false, retain=false 2023-08-10T06:27:52,486 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=455351a9679f34a345fb40a29340fd37, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:27:52,488 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE; OpenRegionProcedure 455351a9679f34a345fb40a29340fd37, server=jenkins-hbase3.apache.org,45101,1691648859755}] 2023-08-10T06:27:52,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-08-10T06:27:52,645 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(130): Open ns2:test-16916488687341,,1691648871482.455351a9679f34a345fb40a29340fd37. 2023-08-10T06:27:52,646 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7254): Opening region: {ENCODED => 455351a9679f34a345fb40a29340fd37, NAME => 'ns2:test-16916488687341,,1691648871482.455351a9679f34a345fb40a29340fd37.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:27:52,646 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:27:52,646 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16916488687341 455351a9679f34a345fb40a29340fd37 2023-08-10T06:27:52,646 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(888): Instantiated ns2:test-16916488687341,,1691648871482.455351a9679f34a345fb40a29340fd37.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:27:52,646 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7296): checking encryption for 455351a9679f34a345fb40a29340fd37 2023-08-10T06:27:52,646 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7299): checking classloading for 455351a9679f34a345fb40a29340fd37 2023-08-10T06:27:52,652 INFO [StoreOpener-455351a9679f34a345fb40a29340fd37-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 455351a9679f34a345fb40a29340fd37 2023-08-10T06:27:52,656 INFO [StoreOpener-455351a9679f34a345fb40a29340fd37-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 455351a9679f34a345fb40a29340fd37 columnFamilyName f 2023-08-10T06:27:52,656 DEBUG [StoreOpener-455351a9679f34a345fb40a29340fd37-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:27:52,657 INFO [StoreOpener-455351a9679f34a345fb40a29340fd37-1 {}] regionserver.HStore(324): Store=455351a9679f34a345fb40a29340fd37/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:27:52,658 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/455351a9679f34a345fb40a29340fd37 2023-08-10T06:27:52,658 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/455351a9679f34a345fb40a29340fd37 2023-08-10T06:27:52,661 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1076): writing seq id for 455351a9679f34a345fb40a29340fd37 2023-08-10T06:27:52,668 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/455351a9679f34a345fb40a29340fd37/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:27:52,669 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): Opened 455351a9679f34a345fb40a29340fd37; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11729800640, jitterRate=0.09242281317710876}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-08-10T06:27:52,670 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(992): Region open journal for 455351a9679f34a345fb40a29340fd37: 2023-08-10T06:27:52,671 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2552): Post open deploy tasks for ns2:test-16916488687341,,1691648871482.455351a9679f34a345fb40a29340fd37., pid=18, masterSystemTime=1691648872641 2023-08-10T06:27:52,674 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2579): Finished post open deploy task for ns2:test-16916488687341,,1691648871482.455351a9679f34a345fb40a29340fd37. 2023-08-10T06:27:52,674 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(158): Opened ns2:test-16916488687341,,1691648871482.455351a9679f34a345fb40a29340fd37. 2023-08-10T06:27:52,675 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=455351a9679f34a345fb40a29340fd37, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:27:52,680 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=18, resume processing ppid=17 2023-08-10T06:27:52,680 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=18, ppid=17, state=SUCCESS; OpenRegionProcedure 455351a9679f34a345fb40a29340fd37, server=jenkins-hbase3.apache.org,45101,1691648859755 in 189 msec 2023-08-10T06:27:52,682 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=17, resume processing ppid=16 2023-08-10T06:27:52,682 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=17, ppid=16, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-16916488687341, region=455351a9679f34a345fb40a29340fd37, ASSIGN in 349 msec 2023-08-10T06:27:52,683 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns2:test-16916488687341 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-08-10T06:27:52,684 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16916488687341","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648872683"}]},"ts":"1691648872683"} 2023-08-10T06:27:52,686 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16916488687341, state=ENABLED in hbase:meta 2023-08-10T06:27:52,689 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16916488687341 execute state=CREATE_TABLE_POST_OPERATION 2023-08-10T06:27:52,690 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=16, state=SUCCESS; CreateTableProcedure table=ns2:test-16916488687341 in 1.2060 sec 2023-08-10T06:27:53,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-08-10T06:27:53,598 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: ns2:test-16916488687341, procId: 16 completed 2023-08-10T06:27:53,603 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HRegion(8210): writing data to region ns2:test-16916488687341,,1691648871482.455351a9679f34a345fb40a29340fd37. with WAL disabled. Data may be lost in the event of a crash. 2023-08-10T06:27:53,695 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'ns3:test-16916488687342', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:27:53,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns3:test-16916488687342 2023-08-10T06:27:53,699 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16916488687342 execute state=CREATE_TABLE_PRE_OPERATION 2023-08-10T06:27:53,699 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:27:53,699 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns3" qualifier: "test-16916488687342" procId is: 19 2023-08-10T06:27:53,700 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns3:test-16916488687342 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-08-10T06:27:53,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-08-10T06:27:53,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-08-10T06:27:54,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-08-10T06:27:54,118 INFO [RegionOpenAndInit-ns3:test-16916488687342-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 9830345ee6e7e7f9278f341f1011836d, NAME => 'ns3:test-16916488687342,,1691648873695.9830345ee6e7e7f9278f341f1011836d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-16916488687342', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e 2023-08-10T06:27:54,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-08-10T06:27:54,420 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test-1691648868734' 2023-08-10T06:27:54,421 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:test-16916488687341' 2023-08-10T06:27:54,536 DEBUG [RegionOpenAndInit-ns3:test-16916488687342-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:test-16916488687342,,1691648873695.9830345ee6e7e7f9278f341f1011836d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:27:54,536 DEBUG [RegionOpenAndInit-ns3:test-16916488687342-pool-0 {}] regionserver.HRegion(1668): Closing 9830345ee6e7e7f9278f341f1011836d, disabling compactions & flushes 2023-08-10T06:27:54,536 INFO [RegionOpenAndInit-ns3:test-16916488687342-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:test-16916488687342,,1691648873695.9830345ee6e7e7f9278f341f1011836d. 2023-08-10T06:27:54,536 DEBUG [RegionOpenAndInit-ns3:test-16916488687342-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16916488687342,,1691648873695.9830345ee6e7e7f9278f341f1011836d. 2023-08-10T06:27:54,536 DEBUG [RegionOpenAndInit-ns3:test-16916488687342-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16916488687342,,1691648873695.9830345ee6e7e7f9278f341f1011836d. after waiting 0 ms 2023-08-10T06:27:54,536 DEBUG [RegionOpenAndInit-ns3:test-16916488687342-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16916488687342,,1691648873695.9830345ee6e7e7f9278f341f1011836d. 2023-08-10T06:27:54,536 INFO [RegionOpenAndInit-ns3:test-16916488687342-pool-0 {}] regionserver.HRegion(1909): Closed ns3:test-16916488687342,,1691648873695.9830345ee6e7e7f9278f341f1011836d. 2023-08-10T06:27:54,536 DEBUG [RegionOpenAndInit-ns3:test-16916488687342-pool-0 {}] regionserver.HRegion(1622): Region close journal for 9830345ee6e7e7f9278f341f1011836d: 2023-08-10T06:27:54,538 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns3:test-16916488687342 execute state=CREATE_TABLE_ADD_TO_META 2023-08-10T06:27:54,538 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:test-16916488687342,,1691648873695.9830345ee6e7e7f9278f341f1011836d.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1691648874538"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691648874538"}]},"ts":"1691648874538"} 2023-08-10T06:27:54,541 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:27:54,542 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns3:test-16916488687342 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-08-10T06:27:54,542 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16916488687342","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648874542"}]},"ts":"1691648874542"} 2023-08-10T06:27:54,545 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16916488687342, state=ENABLING in hbase:meta 2023-08-10T06:27:54,551 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16916488687342, region=9830345ee6e7e7f9278f341f1011836d, ASSIGN}] 2023-08-10T06:27:54,554 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16916488687342, region=9830345ee6e7e7f9278f341f1011836d, ASSIGN 2023-08-10T06:27:54,555 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:test-16916488687342, region=9830345ee6e7e7f9278f341f1011836d, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45101,1691648859755; forceNewPlan=false, retain=false 2023-08-10T06:27:54,706 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=9830345ee6e7e7f9278f341f1011836d, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:27:54,710 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE; OpenRegionProcedure 9830345ee6e7e7f9278f341f1011836d, server=jenkins-hbase3.apache.org,45101,1691648859755}] 2023-08-10T06:27:54,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-08-10T06:27:54,867 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(130): Open ns3:test-16916488687342,,1691648873695.9830345ee6e7e7f9278f341f1011836d. 2023-08-10T06:27:54,867 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7254): Opening region: {ENCODED => 9830345ee6e7e7f9278f341f1011836d, NAME => 'ns3:test-16916488687342,,1691648873695.9830345ee6e7e7f9278f341f1011836d.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:27:54,868 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:27:54,868 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16916488687342 9830345ee6e7e7f9278f341f1011836d 2023-08-10T06:27:54,868 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(888): Instantiated ns3:test-16916488687342,,1691648873695.9830345ee6e7e7f9278f341f1011836d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:27:54,868 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7296): checking encryption for 9830345ee6e7e7f9278f341f1011836d 2023-08-10T06:27:54,868 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7299): checking classloading for 9830345ee6e7e7f9278f341f1011836d 2023-08-10T06:27:54,870 INFO [StoreOpener-9830345ee6e7e7f9278f341f1011836d-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 9830345ee6e7e7f9278f341f1011836d 2023-08-10T06:27:54,872 INFO [StoreOpener-9830345ee6e7e7f9278f341f1011836d-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9830345ee6e7e7f9278f341f1011836d columnFamilyName f 2023-08-10T06:27:54,872 DEBUG [StoreOpener-9830345ee6e7e7f9278f341f1011836d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:27:54,873 INFO [StoreOpener-9830345ee6e7e7f9278f341f1011836d-1 {}] regionserver.HStore(324): Store=9830345ee6e7e7f9278f341f1011836d/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:27:54,874 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns3/test-16916488687342/9830345ee6e7e7f9278f341f1011836d 2023-08-10T06:27:54,874 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns3/test-16916488687342/9830345ee6e7e7f9278f341f1011836d 2023-08-10T06:27:54,877 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1076): writing seq id for 9830345ee6e7e7f9278f341f1011836d 2023-08-10T06:27:54,880 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns3/test-16916488687342/9830345ee6e7e7f9278f341f1011836d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:27:54,881 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): Opened 9830345ee6e7e7f9278f341f1011836d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11477338080, jitterRate=0.06891040503978729}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-08-10T06:27:54,882 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(992): Region open journal for 9830345ee6e7e7f9278f341f1011836d: 2023-08-10T06:27:54,883 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2552): Post open deploy tasks for ns3:test-16916488687342,,1691648873695.9830345ee6e7e7f9278f341f1011836d., pid=21, masterSystemTime=1691648874862 2023-08-10T06:27:54,886 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2579): Finished post open deploy task for ns3:test-16916488687342,,1691648873695.9830345ee6e7e7f9278f341f1011836d. 2023-08-10T06:27:54,886 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(158): Opened ns3:test-16916488687342,,1691648873695.9830345ee6e7e7f9278f341f1011836d. 2023-08-10T06:27:54,886 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=9830345ee6e7e7f9278f341f1011836d, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:27:54,891 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=21, resume processing ppid=20 2023-08-10T06:27:54,891 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=21, ppid=20, state=SUCCESS; OpenRegionProcedure 9830345ee6e7e7f9278f341f1011836d, server=jenkins-hbase3.apache.org,45101,1691648859755 in 179 msec 2023-08-10T06:27:54,893 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=20, resume processing ppid=19 2023-08-10T06:27:54,894 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=20, ppid=19, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-16916488687342, region=9830345ee6e7e7f9278f341f1011836d, ASSIGN in 340 msec 2023-08-10T06:27:54,894 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns3:test-16916488687342 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-08-10T06:27:54,895 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16916488687342","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648874894"}]},"ts":"1691648874894"} 2023-08-10T06:27:54,896 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16916488687342, state=ENABLED in hbase:meta 2023-08-10T06:27:54,899 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16916488687342 execute state=CREATE_TABLE_POST_OPERATION 2023-08-10T06:27:54,901 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=19, state=SUCCESS; CreateTableProcedure table=ns3:test-16916488687342 in 1.2040 sec 2023-08-10T06:27:55,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-08-10T06:27:55,809 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: ns3:test-16916488687342, procId: 19 completed 2023-08-10T06:27:55,809 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns3:test-16916488687342 get assigned. Timeout = 60000ms 2023-08-10T06:27:55,812 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-08-10T06:27:55,816 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.HBaseTestingUtility(3581): All regions for table ns3:test-16916488687342 assigned to meta. Checking AM states. 2023-08-10T06:27:55,817 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-08-10T06:27:55,817 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.HBaseTestingUtility(3601): All regions for table ns3:test-16916488687342 assigned. 2023-08-10T06:27:55,819 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'ns4:test-16916488687343', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:27:55,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns4:test-16916488687343 2023-08-10T06:27:55,823 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16916488687343 execute state=CREATE_TABLE_PRE_OPERATION 2023-08-10T06:27:55,823 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:27:55,823 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns4" qualifier: "test-16916488687343" procId is: 22 2023-08-10T06:27:55,824 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns4:test-16916488687343 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-08-10T06:27:55,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-08-10T06:27:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-08-10T06:27:55,929 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-08-10T06:27:56,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-08-10T06:27:56,237 INFO [RegionOpenAndInit-ns4:test-16916488687343-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 285dade39f9f097a949721d4ab45d071, NAME => 'ns4:test-16916488687343,,1691648875819.285dade39f9f097a949721d4ab45d071.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns4:test-16916488687343', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e 2023-08-10T06:27:56,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-08-10T06:27:56,651 DEBUG [RegionOpenAndInit-ns4:test-16916488687343-pool-0 {}] regionserver.HRegion(888): Instantiated ns4:test-16916488687343,,1691648875819.285dade39f9f097a949721d4ab45d071.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:27:56,651 DEBUG [RegionOpenAndInit-ns4:test-16916488687343-pool-0 {}] regionserver.HRegion(1668): Closing 285dade39f9f097a949721d4ab45d071, disabling compactions & flushes 2023-08-10T06:27:56,652 INFO [RegionOpenAndInit-ns4:test-16916488687343-pool-0 {}] regionserver.HRegion(1690): Closing region ns4:test-16916488687343,,1691648875819.285dade39f9f097a949721d4ab45d071. 2023-08-10T06:27:56,652 DEBUG [RegionOpenAndInit-ns4:test-16916488687343-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16916488687343,,1691648875819.285dade39f9f097a949721d4ab45d071. 2023-08-10T06:27:56,652 DEBUG [RegionOpenAndInit-ns4:test-16916488687343-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16916488687343,,1691648875819.285dade39f9f097a949721d4ab45d071. after waiting 0 ms 2023-08-10T06:27:56,652 DEBUG [RegionOpenAndInit-ns4:test-16916488687343-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16916488687343,,1691648875819.285dade39f9f097a949721d4ab45d071. 2023-08-10T06:27:56,652 INFO [RegionOpenAndInit-ns4:test-16916488687343-pool-0 {}] regionserver.HRegion(1909): Closed ns4:test-16916488687343,,1691648875819.285dade39f9f097a949721d4ab45d071. 2023-08-10T06:27:56,652 DEBUG [RegionOpenAndInit-ns4:test-16916488687343-pool-0 {}] regionserver.HRegion(1622): Region close journal for 285dade39f9f097a949721d4ab45d071: 2023-08-10T06:27:56,653 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns4:test-16916488687343 execute state=CREATE_TABLE_ADD_TO_META 2023-08-10T06:27:56,654 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns4:test-16916488687343,,1691648875819.285dade39f9f097a949721d4ab45d071.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1691648876654"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691648876654"}]},"ts":"1691648876654"} 2023-08-10T06:27:56,656 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:27:56,657 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns4:test-16916488687343 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-08-10T06:27:56,658 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16916488687343","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648876658"}]},"ts":"1691648876658"} 2023-08-10T06:27:56,660 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16916488687343, state=ENABLING in hbase:meta 2023-08-10T06:27:56,664 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16916488687343, region=285dade39f9f097a949721d4ab45d071, ASSIGN}] 2023-08-10T06:27:56,667 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16916488687343, region=285dade39f9f097a949721d4ab45d071, ASSIGN 2023-08-10T06:27:56,668 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns4:test-16916488687343, region=285dade39f9f097a949721d4ab45d071, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45101,1691648859755; forceNewPlan=false, retain=false 2023-08-10T06:27:56,762 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-16916488687342 2023-08-10T06:27:56,762 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns3_table_test-16916488687342 Metrics about Tables on a single HBase RegionServer 2023-08-10T06:27:56,763 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-16916488687341 2023-08-10T06:27:56,763 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_test-16916488687341 Metrics about Tables on a single HBase RegionServer 2023-08-10T06:27:56,764 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1691648868734 2023-08-10T06:27:56,764 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test-1691648868734 Metrics about Tables on a single HBase RegionServer 2023-08-10T06:27:56,820 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=285dade39f9f097a949721d4ab45d071, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:27:56,822 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE; OpenRegionProcedure 285dade39f9f097a949721d4ab45d071, server=jenkins-hbase3.apache.org,45101,1691648859755}] 2023-08-10T06:27:56,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-08-10T06:27:56,980 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(130): Open ns4:test-16916488687343,,1691648875819.285dade39f9f097a949721d4ab45d071. 2023-08-10T06:27:56,980 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7254): Opening region: {ENCODED => 285dade39f9f097a949721d4ab45d071, NAME => 'ns4:test-16916488687343,,1691648875819.285dade39f9f097a949721d4ab45d071.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:27:56,980 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:27:56,980 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16916488687343 285dade39f9f097a949721d4ab45d071 2023-08-10T06:27:56,981 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(888): Instantiated ns4:test-16916488687343,,1691648875819.285dade39f9f097a949721d4ab45d071.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:27:56,981 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7296): checking encryption for 285dade39f9f097a949721d4ab45d071 2023-08-10T06:27:56,981 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7299): checking classloading for 285dade39f9f097a949721d4ab45d071 2023-08-10T06:27:56,983 INFO [StoreOpener-285dade39f9f097a949721d4ab45d071-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 285dade39f9f097a949721d4ab45d071 2023-08-10T06:27:56,984 INFO [StoreOpener-285dade39f9f097a949721d4ab45d071-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 285dade39f9f097a949721d4ab45d071 columnFamilyName f 2023-08-10T06:27:56,984 DEBUG [StoreOpener-285dade39f9f097a949721d4ab45d071-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:27:56,985 INFO [StoreOpener-285dade39f9f097a949721d4ab45d071-1 {}] regionserver.HStore(324): Store=285dade39f9f097a949721d4ab45d071/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:27:56,986 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns4/test-16916488687343/285dade39f9f097a949721d4ab45d071 2023-08-10T06:27:56,987 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns4/test-16916488687343/285dade39f9f097a949721d4ab45d071 2023-08-10T06:27:56,990 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1076): writing seq id for 285dade39f9f097a949721d4ab45d071 2023-08-10T06:27:56,993 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns4/test-16916488687343/285dade39f9f097a949721d4ab45d071/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:27:56,993 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): Opened 285dade39f9f097a949721d4ab45d071; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10477070240, jitterRate=-0.02424679696559906}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-08-10T06:27:56,994 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(992): Region open journal for 285dade39f9f097a949721d4ab45d071: 2023-08-10T06:27:56,995 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2552): Post open deploy tasks for ns4:test-16916488687343,,1691648875819.285dade39f9f097a949721d4ab45d071., pid=24, masterSystemTime=1691648876975 2023-08-10T06:27:56,997 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2579): Finished post open deploy task for ns4:test-16916488687343,,1691648875819.285dade39f9f097a949721d4ab45d071. 2023-08-10T06:27:56,997 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(158): Opened ns4:test-16916488687343,,1691648875819.285dade39f9f097a949721d4ab45d071. 2023-08-10T06:27:56,998 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=285dade39f9f097a949721d4ab45d071, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:27:57,002 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=24, resume processing ppid=23 2023-08-10T06:27:57,002 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=24, ppid=23, state=SUCCESS; OpenRegionProcedure 285dade39f9f097a949721d4ab45d071, server=jenkins-hbase3.apache.org,45101,1691648859755 in 178 msec 2023-08-10T06:27:57,005 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=23, resume processing ppid=22 2023-08-10T06:27:57,005 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=23, ppid=22, state=SUCCESS; TransitRegionStateProcedure table=ns4:test-16916488687343, region=285dade39f9f097a949721d4ab45d071, ASSIGN in 338 msec 2023-08-10T06:27:57,006 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns4:test-16916488687343 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-08-10T06:27:57,006 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16916488687343","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648877006"}]},"ts":"1691648877006"} 2023-08-10T06:27:57,008 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16916488687343, state=ENABLED in hbase:meta 2023-08-10T06:27:57,012 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16916488687343 execute state=CREATE_TABLE_POST_OPERATION 2023-08-10T06:27:57,013 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=22, state=SUCCESS; CreateTableProcedure table=ns4:test-16916488687343 in 1.1930 sec 2023-08-10T06:27:57,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-08-10T06:27:57,932 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: ns4:test-16916488687343, procId: 22 completed 2023-08-10T06:27:57,932 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns4:test-16916488687343 get assigned. Timeout = 60000ms 2023-08-10T06:27:57,932 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-08-10T06:27:57,936 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.HBaseTestingUtility(3581): All regions for table ns4:test-16916488687343 assigned to meta. Checking AM states. 2023-08-10T06:27:57,936 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-08-10T06:27:57,936 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.HBaseTestingUtility(3601): All regions for table ns4:test-16916488687343 assigned. 2023-08-10T06:27:57,937 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x21f31315 to 127.0.0.1:61704 2023-08-10T06:27:57,937 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:27:57,978 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreMultipleOverwrite Thread=582, OpenFileDescriptor=660, MaxFileDescriptor=60000, SystemLoadAverage=208, ProcessCount=171, AvailableMemoryMB=2147 2023-08-10T06:27:57,978 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(130): Thread=582 is superior to 500 2023-08-10T06:27:57,980 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] backup.TestFullRestore(216): create full backup image on multiple tables 2023-08-10T06:27:57,981 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x59568595 to 127.0.0.1:61704 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:27:57,986 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b4c8d73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:27:57,995 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:27:57,997 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:44752, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:27:58,010 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:27:58,011 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:33664, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:27:58,021 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'backup'} 2023-08-10T06:27:58,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=25, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=backup 2023-08-10T06:27:58,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=25 2023-08-10T06:27:58,035 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-08-10T06:27:58,039 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=25, state=SUCCESS; CreateNamespaceProcedure, namespace=backup in 16 msec 2023-08-10T06:27:58,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=25 2023-08-10T06:27:58,143 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'backup:system', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:27:58,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system 2023-08-10T06:27:58,147 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_PRE_OPERATION 2023-08-10T06:27:58,147 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:27:58,147 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system" procId is: 26 2023-08-10T06:27:58,148 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-08-10T06:27:58,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=26 2023-08-10T06:27:58,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=26 2023-08-10T06:27:58,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=26 2023-08-10T06:27:58,566 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => b9c493f2c9fbef45de19e8268a534e72, NAME => 'backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e 2023-08-10T06:27:58,577 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:27:58,578 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1668): Closing b9c493f2c9fbef45de19e8268a534e72, disabling compactions & flushes 2023-08-10T06:27:58,578 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. 2023-08-10T06:27:58,578 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. 2023-08-10T06:27:58,578 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. after waiting 0 ms 2023-08-10T06:27:58,578 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. 2023-08-10T06:27:58,578 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1909): Closed backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. 2023-08-10T06:27:58,578 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1622): Region close journal for b9c493f2c9fbef45de19e8268a534e72: 2023-08-10T06:27:58,580 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ADD_TO_META 2023-08-10T06:27:58,580 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1691648878580"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691648878580"}]},"ts":"1691648878580"} 2023-08-10T06:27:58,582 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:27:58,583 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-08-10T06:27:58,583 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648878583"}]},"ts":"1691648878583"} 2023-08-10T06:27:58,585 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-08-10T06:27:58,589 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=b9c493f2c9fbef45de19e8268a534e72, ASSIGN}] 2023-08-10T06:27:58,591 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=b9c493f2c9fbef45de19e8268a534e72, ASSIGN 2023-08-10T06:27:58,592 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=b9c493f2c9fbef45de19e8268a534e72, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45101,1691648859755; forceNewPlan=false, retain=false 2023-08-10T06:27:58,743 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=b9c493f2c9fbef45de19e8268a534e72, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:27:58,746 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE; OpenRegionProcedure b9c493f2c9fbef45de19e8268a534e72, server=jenkins-hbase3.apache.org,45101,1691648859755}] 2023-08-10T06:27:58,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=26 2023-08-10T06:27:58,903 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(130): Open backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. 2023-08-10T06:27:58,903 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7254): Opening region: {ENCODED => b9c493f2c9fbef45de19e8268a534e72, NAME => 'backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:27:58,904 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:27:58,904 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:27:58,904 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(888): Instantiated backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:27:58,904 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7296): checking encryption for b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:27:58,904 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7299): checking classloading for b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:27:58,905 INFO [StoreOpener-b9c493f2c9fbef45de19e8268a534e72-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:27:58,908 INFO [StoreOpener-b9c493f2c9fbef45de19e8268a534e72-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b9c493f2c9fbef45de19e8268a534e72 columnFamilyName meta 2023-08-10T06:27:58,908 DEBUG [StoreOpener-b9c493f2c9fbef45de19e8268a534e72-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:27:58,909 INFO [StoreOpener-b9c493f2c9fbef45de19e8268a534e72-1 {}] regionserver.HStore(324): Store=b9c493f2c9fbef45de19e8268a534e72/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:27:58,909 INFO [StoreOpener-b9c493f2c9fbef45de19e8268a534e72-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:27:58,911 INFO [StoreOpener-b9c493f2c9fbef45de19e8268a534e72-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b9c493f2c9fbef45de19e8268a534e72 columnFamilyName session 2023-08-10T06:27:58,911 DEBUG [StoreOpener-b9c493f2c9fbef45de19e8268a534e72-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:27:58,911 INFO [StoreOpener-b9c493f2c9fbef45de19e8268a534e72-1 {}] regionserver.HStore(324): Store=b9c493f2c9fbef45de19e8268a534e72/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:27:58,912 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:27:58,913 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:27:58,915 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-08-10T06:27:58,917 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1076): writing seq id for b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:27:58,919 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:27:58,920 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): Opened b9c493f2c9fbef45de19e8268a534e72; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9806490240, jitterRate=-0.08669942617416382}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-08-10T06:27:58,921 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(992): Region open journal for b9c493f2c9fbef45de19e8268a534e72: 2023-08-10T06:27:58,922 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2552): Post open deploy tasks for backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72., pid=28, masterSystemTime=1691648878898 2023-08-10T06:27:58,924 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2579): Finished post open deploy task for backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. 2023-08-10T06:27:58,924 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(158): Opened backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. 2023-08-10T06:27:58,925 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=b9c493f2c9fbef45de19e8268a534e72, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:27:58,929 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=28, resume processing ppid=27 2023-08-10T06:27:58,929 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=28, ppid=27, state=SUCCESS; OpenRegionProcedure b9c493f2c9fbef45de19e8268a534e72, server=jenkins-hbase3.apache.org,45101,1691648859755 in 181 msec 2023-08-10T06:27:58,931 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=27, resume processing ppid=26 2023-08-10T06:27:58,931 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=27, ppid=26, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=b9c493f2c9fbef45de19e8268a534e72, ASSIGN in 340 msec 2023-08-10T06:27:58,932 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-08-10T06:27:58,932 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648878932"}]},"ts":"1691648878932"} 2023-08-10T06:27:58,933 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-08-10T06:27:58,937 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_POST_OPERATION 2023-08-10T06:27:58,939 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=26, state=SUCCESS; CreateTableProcedure table=backup:system in 794 msec 2023-08-10T06:27:59,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=26 2023-08-10T06:27:59,256 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: backup:system, procId: 26 completed 2023-08-10T06:27:59,270 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'backup:system_bulk', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:27:59,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system_bulk 2023-08-10T06:27:59,274 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_PRE_OPERATION 2023-08-10T06:27:59,274 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:27:59,274 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system_bulk" procId is: 29 2023-08-10T06:27:59,275 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-08-10T06:27:59,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-08-10T06:27:59,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-08-10T06:27:59,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-08-10T06:27:59,688 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 12a431f46365339f8fdb6b1e8b7a04c1, NAME => 'backup:system_bulk,,1691648879270.12a431f46365339f8fdb6b1e8b7a04c1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e 2023-08-10T06:27:59,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-08-10T06:28:00,101 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1691648879270.12a431f46365339f8fdb6b1e8b7a04c1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:28:00,101 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1668): Closing 12a431f46365339f8fdb6b1e8b7a04c1, disabling compactions & flushes 2023-08-10T06:28:00,101 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1691648879270.12a431f46365339f8fdb6b1e8b7a04c1. 2023-08-10T06:28:00,101 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1691648879270.12a431f46365339f8fdb6b1e8b7a04c1. 2023-08-10T06:28:00,101 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1691648879270.12a431f46365339f8fdb6b1e8b7a04c1. after waiting 0 ms 2023-08-10T06:28:00,101 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1691648879270.12a431f46365339f8fdb6b1e8b7a04c1. 2023-08-10T06:28:00,101 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1909): Closed backup:system_bulk,,1691648879270.12a431f46365339f8fdb6b1e8b7a04c1. 2023-08-10T06:28:00,101 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1622): Region close journal for 12a431f46365339f8fdb6b1e8b7a04c1: 2023-08-10T06:28:00,102 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ADD_TO_META 2023-08-10T06:28:00,103 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system_bulk,,1691648879270.12a431f46365339f8fdb6b1e8b7a04c1.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1691648880103"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691648880103"}]},"ts":"1691648880103"} 2023-08-10T06:28:00,105 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:28:00,106 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-08-10T06:28:00,106 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648880106"}]},"ts":"1691648880106"} 2023-08-10T06:28:00,108 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLING in hbase:meta 2023-08-10T06:28:00,112 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=12a431f46365339f8fdb6b1e8b7a04c1, ASSIGN}] 2023-08-10T06:28:00,114 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=12a431f46365339f8fdb6b1e8b7a04c1, ASSIGN 2023-08-10T06:28:00,115 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system_bulk, region=12a431f46365339f8fdb6b1e8b7a04c1, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45101,1691648859755; forceNewPlan=false, retain=false 2023-08-10T06:28:00,267 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=12a431f46365339f8fdb6b1e8b7a04c1, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:00,271 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE; OpenRegionProcedure 12a431f46365339f8fdb6b1e8b7a04c1, server=jenkins-hbase3.apache.org,45101,1691648859755}] 2023-08-10T06:28:00,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-08-10T06:28:00,422 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-16916488687342' 2023-08-10T06:28:00,422 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns4:test-16916488687343' 2023-08-10T06:28:00,422 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-08-10T06:28:00,427 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(130): Open backup:system_bulk,,1691648879270.12a431f46365339f8fdb6b1e8b7a04c1. 2023-08-10T06:28:00,427 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7254): Opening region: {ENCODED => 12a431f46365339f8fdb6b1e8b7a04c1, NAME => 'backup:system_bulk,,1691648879270.12a431f46365339f8fdb6b1e8b7a04c1.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:28:00,428 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:28:00,428 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system_bulk 12a431f46365339f8fdb6b1e8b7a04c1 2023-08-10T06:28:00,428 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1691648879270.12a431f46365339f8fdb6b1e8b7a04c1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:28:00,428 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7296): checking encryption for 12a431f46365339f8fdb6b1e8b7a04c1 2023-08-10T06:28:00,428 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7299): checking classloading for 12a431f46365339f8fdb6b1e8b7a04c1 2023-08-10T06:28:00,430 INFO [StoreOpener-12a431f46365339f8fdb6b1e8b7a04c1-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 12a431f46365339f8fdb6b1e8b7a04c1 2023-08-10T06:28:00,432 INFO [StoreOpener-12a431f46365339f8fdb6b1e8b7a04c1-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12a431f46365339f8fdb6b1e8b7a04c1 columnFamilyName meta 2023-08-10T06:28:00,432 DEBUG [StoreOpener-12a431f46365339f8fdb6b1e8b7a04c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:28:00,432 INFO [StoreOpener-12a431f46365339f8fdb6b1e8b7a04c1-1 {}] regionserver.HStore(324): Store=12a431f46365339f8fdb6b1e8b7a04c1/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:28:00,432 INFO [StoreOpener-12a431f46365339f8fdb6b1e8b7a04c1-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 12a431f46365339f8fdb6b1e8b7a04c1 2023-08-10T06:28:00,434 INFO [StoreOpener-12a431f46365339f8fdb6b1e8b7a04c1-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12a431f46365339f8fdb6b1e8b7a04c1 columnFamilyName session 2023-08-10T06:28:00,434 DEBUG [StoreOpener-12a431f46365339f8fdb6b1e8b7a04c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:28:00,435 INFO [StoreOpener-12a431f46365339f8fdb6b1e8b7a04c1-1 {}] regionserver.HStore(324): Store=12a431f46365339f8fdb6b1e8b7a04c1/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:28:00,436 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system_bulk/12a431f46365339f8fdb6b1e8b7a04c1 2023-08-10T06:28:00,437 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system_bulk/12a431f46365339f8fdb6b1e8b7a04c1 2023-08-10T06:28:00,438 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system_bulk descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-08-10T06:28:00,441 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1076): writing seq id for 12a431f46365339f8fdb6b1e8b7a04c1 2023-08-10T06:28:00,444 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system_bulk/12a431f46365339f8fdb6b1e8b7a04c1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:28:00,444 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): Opened 12a431f46365339f8fdb6b1e8b7a04c1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11303101920, jitterRate=0.05268339812755585}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-08-10T06:28:00,445 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(992): Region open journal for 12a431f46365339f8fdb6b1e8b7a04c1: 2023-08-10T06:28:00,446 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2552): Post open deploy tasks for backup:system_bulk,,1691648879270.12a431f46365339f8fdb6b1e8b7a04c1., pid=31, masterSystemTime=1691648880423 2023-08-10T06:28:00,448 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2579): Finished post open deploy task for backup:system_bulk,,1691648879270.12a431f46365339f8fdb6b1e8b7a04c1. 2023-08-10T06:28:00,448 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(158): Opened backup:system_bulk,,1691648879270.12a431f46365339f8fdb6b1e8b7a04c1. 2023-08-10T06:28:00,449 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=12a431f46365339f8fdb6b1e8b7a04c1, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:00,452 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=31, resume processing ppid=30 2023-08-10T06:28:00,453 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=31, ppid=30, state=SUCCESS; OpenRegionProcedure 12a431f46365339f8fdb6b1e8b7a04c1, server=jenkins-hbase3.apache.org,45101,1691648859755 in 180 msec 2023-08-10T06:28:00,455 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=30, resume processing ppid=29 2023-08-10T06:28:00,455 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=30, ppid=29, state=SUCCESS; TransitRegionStateProcedure table=backup:system_bulk, region=12a431f46365339f8fdb6b1e8b7a04c1, ASSIGN in 341 msec 2023-08-10T06:28:00,455 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-08-10T06:28:00,456 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648880455"}]},"ts":"1691648880455"} 2023-08-10T06:28:00,457 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLED in hbase:meta 2023-08-10T06:28:00,460 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_POST_OPERATION 2023-08-10T06:28:00,461 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=29, state=SUCCESS; CreateTableProcedure table=backup:system_bulk in 1.1900 sec 2023-08-10T06:28:01,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-08-10T06:28:01,384 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: backup:system_bulk, procId: 29 completed 2023-08-10T06:28:01,397 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-08-10T06:28:01,457 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:28:01,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691648881458 (current time:1691648881458). 2023-08-10T06:28:01,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:28:01,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-08-10T06:28:01,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0379917b to 127.0.0.1:61704 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:28:01,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30c0b986, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:28:01,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:28:01,471 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:49602, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:28:01,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0379917b to 127.0.0.1:61704 2023-08-10T06:28:01,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:28:01,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:28:01,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=32, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:28:01,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-08-10T06:28:01,488 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:28:01,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-08-10T06:28:01,494 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:28:01,497 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:28:01,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-08-10T06:28:01,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-08-10T06:28:01,909 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:28:01,912 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72}] 2023-08-10T06:28:01,919 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:28:02,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45101 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=33 2023-08-10T06:28:02,080 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. 2023-08-10T06:28:02,082 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2819): Flushing b9c493f2c9fbef45de19e8268a534e72 2/2 column families, dataSize=45 B heapSize=632 B 2023-08-10T06:28:02,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-08-10T06:28:02,157 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/session/e88678d5b4b14eecabac817daf12539e is 49, key is activesession:/session:c/1691648881443/Put/seqid=0 2023-08-10T06:28:02,484 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-08-10T06:28:02,576 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/session/e88678d5b4b14eecabac817daf12539e 2023-08-10T06:28:02,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-08-10T06:28:02,666 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: e88678d5b4b14eecabac817daf12539e 2023-08-10T06:28:02,672 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/session/e88678d5b4b14eecabac817daf12539e as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/e88678d5b4b14eecabac817daf12539e 2023-08-10T06:28:02,683 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/e88678d5b4b14eecabac817daf12539e, entries=1, sequenceid=5, filesize=4.9 K 2023-08-10T06:28:02,688 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(3022): Finished flush of dataSize ~45 B/45, heapSize ~360 B/360, currentSize=0 B/0 for b9c493f2c9fbef45de19e8268a534e72 in 606ms, sequenceid=5, compaction requested=false 2023-08-10T06:28:02,691 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2520): Flush status journal for b9c493f2c9fbef45de19e8268a534e72: 2023-08-10T06:28:02,691 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. for snapshot_backup_system completed. 2023-08-10T06:28:02,693 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72.' region-info for snapshot=snapshot_backup_system 2023-08-10T06:28:02,702 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:28:02,709 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-08-10T06:28:02,709 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/e88678d5b4b14eecabac817daf12539e] hfiles 2023-08-10T06:28:02,709 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/e88678d5b4b14eecabac817daf12539e for snapshot=snapshot_backup_system 2023-08-10T06:28:02,730 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. 2023-08-10T06:28:02,734 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=33 2023-08-10T06:28:02,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster(3994): Remote procedure done, pid=33 2023-08-10T06:28:02,739 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:28:02,741 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:28:02,744 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=33, resume processing ppid=32 2023-08-10T06:28:02,744 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=33, ppid=32, state=SUCCESS; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72 in 830 msec 2023-08-10T06:28:02,744 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:28:02,746 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:28:02,750 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:28:02,750 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-08-10T06:28:02,753 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_backup_system 2023-08-10T06:28:03,192 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:28:03,206 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:28:03,207 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_backup_system 2023-08-10T06:28:03,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-08-10T06:28:04,045 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:28:04,046 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-08-10T06:28:04,047 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=32, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.5650 sec 2023-08-10T06:28:05,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-08-10T06:28:05,599 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: backup:system, procId: 32 completed 2023-08-10T06:28:05,600 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(115): Backup backup_1691648877989 started at 1691648885599. 2023-08-10T06:28:05,621 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(119): Backup session backup_1691648877989 has been started. 2023-08-10T06:28:05,625 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-08-10T06:28:05,635 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-08-10T06:28:05,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-08-10T06:28:05,641 INFO [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-08-10T06:28:05,641 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-08-10T06:28:05,642 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-08-10T06:28:05,642 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-08-10T06:28:05,643 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-08-10T06:28:05,643 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:28:05,644 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:05,644 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45101-0x10060ea6e8b0001, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-08-10T06:28:05,644 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:28:05,644 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:28:05,644 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:05,644 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-08-10T06:28:05,645 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-08-10T06:28:05,645 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45101-0x10060ea6e8b0001, quorum=127.0.0.1:61704, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-08-10T06:28:05,645 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-08-10T06:28:05,645 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:28:05,646 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-08-10T06:28:05,647 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-08-10T06:28:05,647 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-08-10T06:28:05,648 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-08-10T06:28:05,648 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-08-10T06:28:05,648 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-08-10T06:28:05,648 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-08-10T06:28:05,648 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,45101,1691648859755' joining acquired barrier for procedure (rolllog) in zk 2023-08-10T06:28:05,651 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-08-10T06:28:05,651 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:05,651 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:05,651 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:28:05,651 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:28:05,651 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45101-0x10060ea6e8b0001, quorum=127.0.0.1:61704, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-08-10T06:28:05,651 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-08-10T06:28:05,651 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:28:05,652 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:28:05,652 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:28:05,652 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:05,653 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:28:05,653 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,45101,1691648859755' joining acquired barrier for procedure 'rolllog' on coordinator 2023-08-10T06:28:05,653 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@6f160cff[Count = 0] remaining members to acquire global barrier 2023-08-10T06:28:05,653 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-08-10T06:28:05,653 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-08-10T06:28:05,657 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45101-0x10060ea6e8b0001, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-08-10T06:28:05,657 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-08-10T06:28:05,657 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-08-10T06:28:05,657 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-08-10T06:28:05,657 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:05,657 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-08-10T06:28:05,658 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-08-10T06:28:05,658 DEBUG [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:05,658 INFO [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1691648862187 highest: 1691648862187 on jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:05,658 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta:.meta(num 1691648862717) roll requested 2023-08-10T06:28:05,670 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691648885658.meta, exclude list is [], retry=0 2023-08-10T06:28:05,675 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK] 2023-08-10T06:28:05,678 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691648862717.meta with entries=36, filesize=11.13 KB; new WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691648885658.meta 2023-08-10T06:28:05,679 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK]] 2023-08-10T06:28:05,679 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691648862717.meta is not closed yet, will try archiving it next time 2023-08-10T06:28:05,679 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C45101%2C1691648859755:(num 1691648862187) roll requested 2023-08-10T06:28:05,692 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691648885679, exclude list is [], retry=0 2023-08-10T06:28:05,695 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK] 2023-08-10T06:28:05,698 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691648862187 with entries=19, filesize=4.02 KB; new WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691648885679 2023-08-10T06:28:05,699 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK]] 2023-08-10T06:28:05,699 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691648862187 is not closed yet, will try archiving it next time 2023-08-10T06:28:05,699 DEBUG [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 41 2023-08-10T06:28:05,699 INFO [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1691648885679 on jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:05,700 DEBUG [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:28:05,702 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:54476, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-08-10T06:28:05,721 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-08-10T06:28:05,721 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,45101,1691648859755' in zk 2023-08-10T06:28:05,724 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:05,724 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-08-10T06:28:05,724 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:28:05,724 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:05,725 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-08-10T06:28:05,725 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:28:05,725 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:28:05,726 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:28:05,726 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:28:05,726 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:28:05,727 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:05,727 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:28:05,727 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:28:05,728 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:05,728 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,45101,1691648859755': 2023-08-10T06:28:05,729 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,45101,1691648859755' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-08-10T06:28:05,729 INFO [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-08-10T06:28:05,729 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-08-10T06:28:05,729 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-08-10T06:28:05,729 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-08-10T06:28:05,729 INFO [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-08-10T06:28:05,731 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-08-10T06:28:05,731 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45101-0x10060ea6e8b0001, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-08-10T06:28:05,731 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45101-0x10060ea6e8b0001, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-08-10T06:28:05,731 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-08-10T06:28:05,731 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-08-10T06:28:05,731 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:28:05,731 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:28:05,731 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:28:05,732 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:28:05,732 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:28:05,732 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:28:05,732 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:05,732 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:28:05,732 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:28:05,733 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:28:05,733 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:28:05,734 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:05,734 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:05,734 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:28:05,735 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:28:05,735 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:05,744 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:05,744 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45101-0x10060ea6e8b0001, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-08-10T06:28:05,744 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-08-10T06:28:05,744 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-08-10T06:28:05,744 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:28:05,744 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:05,744 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-08-10T06:28:05,744 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:28:05,744 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-08-10T06:28:05,744 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-08-10T06:28:05,744 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:28:05,745 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-08-10T06:28:05,745 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-08-10T06:28:05,745 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45101-0x10060ea6e8b0001, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-08-10T06:28:05,745 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:28:05,745 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:28:05,748 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] client.HBaseAdmin(2751): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-08-10T06:28:05,748 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] client.HBaseAdmin(2759): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-08-10T06:28:05,848 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] client.HBaseAdmin(2765): Getting current status of procedure from master... 2023-08-10T06:28:05,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-08-10T06:28:05,861 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1691648885860_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } 2023-08-10T06:28:05,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691648885861 (current time:1691648885861). 2023-08-10T06:28:05,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:28:05,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1691648885860_ns2_test-16916488687341 VERSION not specified, setting to 2 2023-08-10T06:28:05,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1aa9a985 to 127.0.0.1:61704 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:28:05,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a52a35d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:28:05,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:28:05,874 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:49608, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:28:05,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1aa9a985 to 127.0.0.1:61704 2023-08-10T06:28:05,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:28:05,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:28:05,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=34, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1691648885860_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } 2023-08-10T06:28:05,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1691648885860_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 }, snapshot procedure id = 34 2023-08-10T06:28:05,879 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1691648885860_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:28:05,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-08-10T06:28:05,880 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1691648885860_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:28:05,883 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1691648885860_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:28:05,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-08-10T06:28:06,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-08-10T06:28:06,295 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1691648885860_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:28:06,295 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure 455351a9679f34a345fb40a29340fd37}] 2023-08-10T06:28:06,298 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure 455351a9679f34a345fb40a29340fd37 2023-08-10T06:28:06,423 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system_bulk' 2023-08-10T06:28:06,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45101 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=35 2023-08-10T06:28:06,450 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-16916488687341,,1691648871482.455351a9679f34a345fb40a29340fd37. 2023-08-10T06:28:06,451 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2819): Flushing 455351a9679f34a345fb40a29340fd37 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-08-10T06:28:06,471 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/455351a9679f34a345fb40a29340fd37/.tmp/f/7dc1c6bf67474a4488e888a4dcbb27b6 is 37, key is row10/f:q1/1691648873613/Put/seqid=0 2023-08-10T06:28:06,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-08-10T06:28:06,762 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-08-10T06:28:06,762 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2023-08-10T06:28:06,763 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2023-08-10T06:28:06,763 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk Metrics about Tables on a single HBase RegionServer 2023-08-10T06:28:06,764 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-16916488687343 2023-08-10T06:28:06,764 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns4_table_test-16916488687343 Metrics about Tables on a single HBase RegionServer 2023-08-10T06:28:06,880 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/455351a9679f34a345fb40a29340fd37/.tmp/f/7dc1c6bf67474a4488e888a4dcbb27b6 2023-08-10T06:28:06,890 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 7dc1c6bf67474a4488e888a4dcbb27b6 2023-08-10T06:28:06,891 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/455351a9679f34a345fb40a29340fd37/.tmp/f/7dc1c6bf67474a4488e888a4dcbb27b6 as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/455351a9679f34a345fb40a29340fd37/f/7dc1c6bf67474a4488e888a4dcbb27b6 2023-08-10T06:28:06,898 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/455351a9679f34a345fb40a29340fd37/f/7dc1c6bf67474a4488e888a4dcbb27b6, entries=99, sequenceid=103, filesize=8.2 K 2023-08-10T06:28:06,900 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 455351a9679f34a345fb40a29340fd37 in 449ms, sequenceid=103, compaction requested=false 2023-08-10T06:28:06,900 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2520): Flush status journal for 455351a9679f34a345fb40a29340fd37: 2023-08-10T06:28:06,900 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-16916488687341,,1691648871482.455351a9679f34a345fb40a29340fd37. for snapshot_1691648885860_ns2_test-16916488687341 completed. 2023-08-10T06:28:06,900 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(240): Storing 'ns2:test-16916488687341,,1691648871482.455351a9679f34a345fb40a29340fd37.' region-info for snapshot=snapshot_1691648885860_ns2_test-16916488687341 2023-08-10T06:28:06,901 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:28:06,901 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/455351a9679f34a345fb40a29340fd37/f/7dc1c6bf67474a4488e888a4dcbb27b6] hfiles 2023-08-10T06:28:06,901 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/455351a9679f34a345fb40a29340fd37/f/7dc1c6bf67474a4488e888a4dcbb27b6 for snapshot=snapshot_1691648885860_ns2_test-16916488687341 2023-08-10T06:28:06,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-08-10T06:28:07,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-16916488687341,,1691648871482.455351a9679f34a345fb40a29340fd37. 2023-08-10T06:28:07,312 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=35 2023-08-10T06:28:07,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster(3994): Remote procedure done, pid=35 2023-08-10T06:28:07,312 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1691648885860_ns2_test-16916488687341 on region 455351a9679f34a345fb40a29340fd37 2023-08-10T06:28:07,314 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure 455351a9679f34a345fb40a29340fd37 2023-08-10T06:28:07,317 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=35, resume processing ppid=34 2023-08-10T06:28:07,317 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1691648885860_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:28:07,317 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=35, ppid=34, state=SUCCESS; SnapshotRegionProcedure 455351a9679f34a345fb40a29340fd37 in 1.0200 sec 2023-08-10T06:28:07,318 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1691648885860_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:28:07,319 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1691648885860_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:28:07,319 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1691648885860_ns2_test-16916488687341 2023-08-10T06:28:07,320 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_1691648885860_ns2_test-16916488687341 2023-08-10T06:28:07,734 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1691648885860_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:28:07,742 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1691648885860_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:28:07,742 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_1691648885860_ns2_test-16916488687341 to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691648885860_ns2_test-16916488687341 2023-08-10T06:28:07,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-08-10T06:28:08,572 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1691648885860_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:28:08,572 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1691648885860_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 }, snapshot procedure id = 34 2023-08-10T06:28:08,573 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=34, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1691648885860_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } in 2.6960 sec 2023-08-10T06:28:09,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-08-10T06:28:09,988 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: ns2:test-16916488687341, procId: 34 completed 2023-08-10T06:28:09,989 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1691648889988_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } 2023-08-10T06:28:09,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691648889989 (current time:1691648889989). 2023-08-10T06:28:09,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:28:09,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1691648889988_ns3_test-16916488687342 VERSION not specified, setting to 2 2023-08-10T06:28:09,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5612c49f to 127.0.0.1:61704 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:28:09,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a0ba59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:28:09,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:28:09,998 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:49624, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:28:09,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5612c49f to 127.0.0.1:61704 2023-08-10T06:28:09,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:28:10,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:28:10,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=36, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1691648889988_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } 2023-08-10T06:28:10,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1691648889988_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 }, snapshot procedure id = 36 2023-08-10T06:28:10,002 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1691648889988_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:28:10,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=36 2023-08-10T06:28:10,004 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1691648889988_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:28:10,006 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1691648889988_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:28:10,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=36 2023-08-10T06:28:10,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=36 2023-08-10T06:28:10,415 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1691648889988_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:28:10,415 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE; SnapshotRegionProcedure 9830345ee6e7e7f9278f341f1011836d}] 2023-08-10T06:28:10,417 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=37, ppid=36, state=RUNNABLE; SnapshotRegionProcedure 9830345ee6e7e7f9278f341f1011836d 2023-08-10T06:28:10,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45101 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=37 2023-08-10T06:28:10,569 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-16916488687342,,1691648873695.9830345ee6e7e7f9278f341f1011836d. 2023-08-10T06:28:10,570 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2520): Flush status journal for 9830345ee6e7e7f9278f341f1011836d: 2023-08-10T06:28:10,571 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-16916488687342,,1691648873695.9830345ee6e7e7f9278f341f1011836d. for snapshot_1691648889988_ns3_test-16916488687342 completed. 2023-08-10T06:28:10,571 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(240): Storing 'ns3:test-16916488687342,,1691648873695.9830345ee6e7e7f9278f341f1011836d.' region-info for snapshot=snapshot_1691648889988_ns3_test-16916488687342 2023-08-10T06:28:10,571 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:28:10,571 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-08-10T06:28:10,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=36 2023-08-10T06:28:10,980 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-16916488687342,,1691648873695.9830345ee6e7e7f9278f341f1011836d. 2023-08-10T06:28:10,980 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=37 2023-08-10T06:28:10,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster(3994): Remote procedure done, pid=37 2023-08-10T06:28:10,981 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1691648889988_ns3_test-16916488687342 on region 9830345ee6e7e7f9278f341f1011836d 2023-08-10T06:28:10,982 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=37, ppid=36, state=RUNNABLE; SnapshotRegionProcedure 9830345ee6e7e7f9278f341f1011836d 2023-08-10T06:28:10,985 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=37, resume processing ppid=36 2023-08-10T06:28:10,986 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1691648889988_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:28:10,986 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=37, ppid=36, state=SUCCESS; SnapshotRegionProcedure 9830345ee6e7e7f9278f341f1011836d in 568 msec 2023-08-10T06:28:10,986 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1691648889988_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:28:10,987 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1691648889988_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:28:10,987 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1691648889988_ns3_test-16916488687342 2023-08-10T06:28:10,988 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_1691648889988_ns3_test-16916488687342 2023-08-10T06:28:11,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=36 2023-08-10T06:28:11,406 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1691648889988_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:28:11,412 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1691648889988_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:28:11,412 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_1691648889988_ns3_test-16916488687342 to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691648889988_ns3_test-16916488687342 2023-08-10T06:28:11,627 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-08-10T06:28:11,629 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:55986, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-08-10T06:28:12,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=36 2023-08-10T06:28:12,239 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1691648889988_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:28:12,239 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1691648889988_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 }, snapshot procedure id = 36 2023-08-10T06:28:12,240 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=36, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1691648889988_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } in 2.2390 sec 2023-08-10T06:28:14,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=36 2023-08-10T06:28:14,112 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: ns3:test-16916488687342, procId: 36 completed 2023-08-10T06:28:14,112 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1691648877989 2023-08-10T06:28:14,112 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-08-10T06:28:14,114 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2023-08-10T06:28:14,116 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1691648877989_ns3:test-16916488687342 2023-08-10T06:28:14,116 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1691648889988_ns3_test-16916488687342 to hdfs://localhost:45379/backupUT/backup_1691648877989/ns3/test-16916488687342/ 2023-08-10T06:28:14,126 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-08-10T06:28:14,164 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:45379, inputRoot=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e 2023-08-10T06:28:14,165 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,33705,1691648856595_-786591061_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:45379/backupUT/backup_1691648877989/ns3/test-16916488687342, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:45379/backupUT/backup_1691648877989/ns3/test-16916488687342/.hbase-snapshot/.tmp/snapshot_1691648889988_ns3_test-16916488687342 2023-08-10T06:28:14,165 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:45379, inputRoot=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e, snapshotDir=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691648889988_ns3_test-16916488687342. 2023-08-10T06:28:14,174 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691648889988_ns3_test-16916488687342 to hdfs://localhost:45379/backupUT/backup_1691648877989/ns3/test-16916488687342/.hbase-snapshot/.tmp/snapshot_1691648889988_ns3_test-16916488687342 2023-08-10T06:28:14,607 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:14,608 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:14,609 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:14,609 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:15,619 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/hadoop-3849864222392941035.jar 2023-08-10T06:28:15,619 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:15,619 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:15,620 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:15,621 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:15,621 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:15,621 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:15,622 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:15,622 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:15,623 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:15,623 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:28:15,624 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-08-10T06:28:15,625 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-08-10T06:28:15,625 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-08-10T06:28:15,626 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-08-10T06:28:15,626 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-08-10T06:28:15,626 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-08-10T06:28:15,627 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-08-10T06:28:15,627 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-08-10T06:28:15,628 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-08-10T06:28:15,628 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-08-10T06:28:15,629 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-08-10T06:28:15,635 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:28:15,635 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:28:15,636 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:28:15,636 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:28:15,637 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:28:15,638 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:28:15,638 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:28:15,703 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-08-10T06:28:15,726 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1691648889988_ns3_test-16916488687342' hfile list 2023-08-10T06:28:15,926 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691648895880/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-08-10T06:28:15,927 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691648895880/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-08-10T06:28:17,012 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-08-10T06:28:17,013 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-08-10T06:28:17,019 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1691648889988_ns3_test-16916488687342 2023-08-10T06:28:17,019 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1691648889988_ns3_test-16916488687342 finished. 2023-08-10T06:28:17,019 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1691648877989_ns2:test-16916488687341 2023-08-10T06:28:17,019 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1691648885860_ns2_test-16916488687341 to hdfs://localhost:45379/backupUT/backup_1691648877989/ns2/test-16916488687341/ 2023-08-10T06:28:17,020 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-08-10T06:28:17,042 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:45379, inputRoot=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e 2023-08-10T06:28:17,042 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,33705,1691648856595_-1445124726_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:45379/backupUT/backup_1691648877989/ns2/test-16916488687341, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:45379/backupUT/backup_1691648877989/ns2/test-16916488687341/.hbase-snapshot/.tmp/snapshot_1691648885860_ns2_test-16916488687341 2023-08-10T06:28:17,042 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:45379, inputRoot=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e, snapshotDir=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691648885860_ns2_test-16916488687341. 2023-08-10T06:28:17,054 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691648885860_ns2_test-16916488687341 to hdfs://localhost:45379/backupUT/backup_1691648877989/ns2/test-16916488687341/.hbase-snapshot/.tmp/snapshot_1691648885860_ns2_test-16916488687341 2023-08-10T06:28:17,477 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:17,477 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:17,478 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:17,478 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:18,461 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/hadoop-1020103124193521143.jar 2023-08-10T06:28:18,462 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:18,462 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:18,462 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:18,463 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:18,463 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:18,464 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:18,464 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:18,464 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:18,465 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:18,466 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:28:18,466 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-08-10T06:28:18,467 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-08-10T06:28:18,467 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-08-10T06:28:18,468 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-08-10T06:28:18,468 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-08-10T06:28:18,469 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-08-10T06:28:18,469 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-08-10T06:28:18,469 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-08-10T06:28:18,470 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-08-10T06:28:18,470 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-08-10T06:28:18,471 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-08-10T06:28:18,471 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:28:18,472 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:28:18,473 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:28:18,473 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:28:18,473 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:28:18,474 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:28:18,475 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:28:18,501 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-08-10T06:28:18,519 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1691648885860_ns2_test-16916488687341' hfile list 2023-08-10T06:28:18,527 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-08-10T06:28:18,611 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691648898575/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-08-10T06:28:18,611 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691648898575/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-08-10T06:28:18,723 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-08-10T06:28:18,753 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/455351a9679f34a345fb40a29340fd37/f/7dc1c6bf67474a4488e888a4dcbb27b6 output=hdfs://localhost:45379/backupUT/backup_1691648877989/ns2/test-16916488687341/archive/data/ns2/test-16916488687341/455351a9679f34a345fb40a29340fd37/f/7dc1c6bf67474a4488e888a4dcbb27b6 2023-08-10T06:28:18,753 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 8.007M/sec 2023-08-10T06:28:19,652 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-08-10T06:28:19,653 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-08-10T06:28:19,662 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1691648885860_ns2_test-16916488687341 2023-08-10T06:28:19,662 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1691648885860_ns2_test-16916488687341 finished. 2023-08-10T06:28:19,662 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(1014): ns3:test-16916488687342 2023-08-10T06:28:19,662 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(1014): ns2:test-16916488687341 2023-08-10T06:28:19,741 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691648877989 2023-08-10T06:28:19,741 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-08-10T06:28:20,156 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:45379/backupUT/backup_1691648877989/.backup.manifest 2023-08-10T06:28:20,157 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691648877989 2023-08-10T06:28:20,157 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-08-10T06:28:20,563 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:45379/backupUT/backup_1691648877989/.backup.manifest 2023-08-10T06:28:20,563 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(358): Backup backup_1691648877989 finished: type=FULL,tablelist=ns3:test-16916488687342;ns2:test-16916488687341,targetRootDir=hdfs://localhost:45379/backupUT,startts=1691648885599,completets=1691648899738,bytescopied=0 2023-08-10T06:28:20,563 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-08-10T06:28:20,563 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1691648889988_ns3_test-16916488687342 2023-08-10T06:28:20,575 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1691648889988_ns3_test-16916488687342" 2023-08-10T06:28:20,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1691648889988_ns3_test-16916488687342 2023-08-10T06:28:20,581 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1691648889988_ns3_test-16916488687342 for backup backup_1691648877989 succeeded. 2023-08-10T06:28:20,581 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1691648885860_ns2_test-16916488687341 2023-08-10T06:28:20,582 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1691648885860_ns2_test-16916488687341" 2023-08-10T06:28:20,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1691648885860_ns2_test-16916488687341 2023-08-10T06:28:20,585 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1691648885860_ns2_test-16916488687341 for backup backup_1691648877989 succeeded. 2023-08-10T06:28:20,586 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-08-10T06:28:20,598 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-08-10T06:28:20,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-08-10T06:28:20,602 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-08-10T06:28:20,610 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-08-10T06:28:20,632 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(376): Backup backup_1691648877989 completed. 2023-08-10T06:28:20,633 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] client.ConnectionImplementation(2114): Closing master protocol: MasterService 2023-08-10T06:28:20,633 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x59568595 to 127.0.0.1:61704 2023-08-10T06:28:20,633 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:28:20,664 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:45379/backupUT/backup_1691648877989/.backup.manifest 2023-08-10T06:28:20,667 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691648877989/.backup.manifest 2023-08-10T06:28:20,667 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:45379/backupUT/backup_1691648877989/.backup.manifest 2023-08-10T06:28:20,670 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691648877989/.backup.manifest 2023-08-10T06:28:20,677 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:45379/backupUT/backup_1691648877989/.backup.manifest 2023-08-10T06:28:20,680 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691648877989/.backup.manifest 2023-08-10T06:28:20,680 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(148): Restoring 'ns2:test-16916488687341' to 'ns2:test-16916488687341' from full backup image hdfs://localhost:45379/backupUT/backup_1691648877989/ns2/test-16916488687341 2023-08-10T06:28:20,692 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] util.RestoreTool(476): Truncating exising target table 'ns2:test-16916488687341', preserving region splits 2023-08-10T06:28:20,693 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] client.HBaseAdmin$18(963): Started disable of ns2:test-16916488687341 2023-08-10T06:28:20,698 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster$12(2663): Client=jenkins//172.31.12.81 disable ns2:test-16916488687341 2023-08-10T06:28:20,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=38, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=ns2:test-16916488687341 2023-08-10T06:28:20,710 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16916488687341","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648900710"}]},"ts":"1691648900710"} 2023-08-10T06:28:20,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=38 2023-08-10T06:28:20,712 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16916488687341, state=DISABLING in hbase:meta 2023-08-10T06:28:20,713 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(293): Set ns2:test-16916488687341 to state=DISABLING 2023-08-10T06:28:20,715 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns2:test-16916488687341, region=455351a9679f34a345fb40a29340fd37, UNASSIGN}] 2023-08-10T06:28:20,717 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns2:test-16916488687341, region=455351a9679f34a345fb40a29340fd37, UNASSIGN 2023-08-10T06:28:20,718 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=39 updating hbase:meta row=455351a9679f34a345fb40a29340fd37, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:20,720 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE; CloseRegionProcedure 455351a9679f34a345fb40a29340fd37, server=jenkins-hbase3.apache.org,45101,1691648859755}] 2023-08-10T06:28:20,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=38 2023-08-10T06:28:20,881 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(119): Close 455351a9679f34a345fb40a29340fd37 2023-08-10T06:28:20,881 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1668): Closing 455351a9679f34a345fb40a29340fd37, disabling compactions & flushes 2023-08-10T06:28:20,881 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1690): Closing region ns2:test-16916488687341,,1691648871482.455351a9679f34a345fb40a29340fd37. 2023-08-10T06:28:20,881 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16916488687341,,1691648871482.455351a9679f34a345fb40a29340fd37. 2023-08-10T06:28:20,881 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16916488687341,,1691648871482.455351a9679f34a345fb40a29340fd37. after waiting 0 ms 2023-08-10T06:28:20,881 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16916488687341,,1691648871482.455351a9679f34a345fb40a29340fd37. 2023-08-10T06:28:20,884 DEBUG [StoreFileCloser-455351a9679f34a345fb40a29340fd37-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 7dc1c6bf67474a4488e888a4dcbb27b6 2023-08-10T06:28:20,890 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/455351a9679f34a345fb40a29340fd37/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2023-08-10T06:28:20,890 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:28:20,891 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1909): Closed ns2:test-16916488687341,,1691648871482.455351a9679f34a345fb40a29340fd37. 2023-08-10T06:28:20,891 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1622): Region close journal for 455351a9679f34a345fb40a29340fd37: 2023-08-10T06:28:20,894 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(163): Closed 455351a9679f34a345fb40a29340fd37 2023-08-10T06:28:20,894 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=39 updating hbase:meta row=455351a9679f34a345fb40a29340fd37, regionState=CLOSED 2023-08-10T06:28:20,898 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=40, resume processing ppid=39 2023-08-10T06:28:20,898 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=40, ppid=39, state=SUCCESS; CloseRegionProcedure 455351a9679f34a345fb40a29340fd37, server=jenkins-hbase3.apache.org,45101,1691648859755 in 176 msec 2023-08-10T06:28:20,901 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=39, resume processing ppid=38 2023-08-10T06:28:20,901 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=39, ppid=38, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-16916488687341, region=455351a9679f34a345fb40a29340fd37, UNASSIGN in 184 msec 2023-08-10T06:28:20,902 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16916488687341","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648900902"}]},"ts":"1691648900902"} 2023-08-10T06:28:20,904 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16916488687341, state=DISABLED in hbase:meta 2023-08-10T06:28:20,906 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(305): Set ns2:test-16916488687341 to state=DISABLED 2023-08-10T06:28:20,909 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=38, state=SUCCESS; DisableTableProcedure table=ns2:test-16916488687341 in 206 msec 2023-08-10T06:28:21,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=38 2023-08-10T06:28:21,016 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] client.HBaseAdmin$TableFuture(3590): Operation: DISABLE, Table Name: ns2:test-16916488687341, procId: 38 completed 2023-08-10T06:28:21,017 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] client.HBaseAdmin$16(843): Started truncating ns2:test-16916488687341 2023-08-10T06:28:21,024 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster$6(2479): Client=jenkins//172.31.12.81 truncate ns2:test-16916488687341 2023-08-10T06:28:21,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=41, state=RUNNABLE:TRUNCATE_TABLE_PRE_OPERATION; TruncateTableProcedure (table=ns2:test-16916488687341 preserveSplits=true) 2023-08-10T06:28:21,036 DEBUG [PEWorker-2 {}] procedure.TruncateTableProcedure(87): waiting for 'ns2:test-16916488687341' regions in transition 2023-08-10T06:28:21,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=41 2023-08-10T06:28:21,045 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/455351a9679f34a345fb40a29340fd37 2023-08-10T06:28:21,050 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/455351a9679f34a345fb40a29340fd37/f, FileablePath, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/455351a9679f34a345fb40a29340fd37/recovered.edits] 2023-08-10T06:28:21,058 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/455351a9679f34a345fb40a29340fd37/f/7dc1c6bf67474a4488e888a4dcbb27b6 to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/archive/data/ns2/test-16916488687341/455351a9679f34a345fb40a29340fd37/f/7dc1c6bf67474a4488e888a4dcbb27b6 2023-08-10T06:28:21,062 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/455351a9679f34a345fb40a29340fd37/recovered.edits/106.seqid to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/archive/data/ns2/test-16916488687341/455351a9679f34a345fb40a29340fd37/recovered.edits/106.seqid 2023-08-10T06:28:21,065 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/455351a9679f34a345fb40a29340fd37 2023-08-10T06:28:21,066 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(293): Archived ns2:test-16916488687341 regions 2023-08-10T06:28:21,080 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of ns2:test-16916488687341 from hbase:meta 2023-08-10T06:28:21,083 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(387): Removing 'ns2:test-16916488687341' descriptor. 2023-08-10T06:28:21,084 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(377): Removing 'ns2:test-16916488687341' from region states. 2023-08-10T06:28:21,084 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns2:test-16916488687341,,1691648871482.455351a9679f34a345fb40a29340fd37.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1691648901084"}]},"ts":"9223372036854775807"} 2023-08-10T06:28:21,086 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-08-10T06:28:21,086 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 455351a9679f34a345fb40a29340fd37, NAME => 'ns2:test-16916488687341,,1691648871482.455351a9679f34a345fb40a29340fd37.', STARTKEY => '', ENDKEY => ''}] 2023-08-10T06:28:21,086 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(381): Marking 'ns2:test-16916488687341' as deleted. 2023-08-10T06:28:21,086 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns2:test-16916488687341","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1691648901086"}]},"ts":"9223372036854775807"} 2023-08-10T06:28:21,095 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1678): Deleted table ns2:test-16916488687341 state from META 2023-08-10T06:28:21,106 INFO [RegionOpenAndInit-ns2:test-16916488687341-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 4897eee4660ec19a1cc212f00aac000c, NAME => 'ns2:test-16916488687341,,1691648901068.4897eee4660ec19a1cc212f00aac000c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-16916488687341', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e 2023-08-10T06:28:21,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=41 2023-08-10T06:28:21,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=41 2023-08-10T06:28:21,515 DEBUG [RegionOpenAndInit-ns2:test-16916488687341-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:test-16916488687341,,1691648901068.4897eee4660ec19a1cc212f00aac000c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:28:21,516 DEBUG [RegionOpenAndInit-ns2:test-16916488687341-pool-0 {}] regionserver.HRegion(1668): Closing 4897eee4660ec19a1cc212f00aac000c, disabling compactions & flushes 2023-08-10T06:28:21,516 INFO [RegionOpenAndInit-ns2:test-16916488687341-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:test-16916488687341,,1691648901068.4897eee4660ec19a1cc212f00aac000c. 2023-08-10T06:28:21,516 DEBUG [RegionOpenAndInit-ns2:test-16916488687341-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16916488687341,,1691648901068.4897eee4660ec19a1cc212f00aac000c. 2023-08-10T06:28:21,516 DEBUG [RegionOpenAndInit-ns2:test-16916488687341-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16916488687341,,1691648901068.4897eee4660ec19a1cc212f00aac000c. after waiting 0 ms 2023-08-10T06:28:21,516 DEBUG [RegionOpenAndInit-ns2:test-16916488687341-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16916488687341,,1691648901068.4897eee4660ec19a1cc212f00aac000c. 2023-08-10T06:28:21,516 INFO [RegionOpenAndInit-ns2:test-16916488687341-pool-0 {}] regionserver.HRegion(1909): Closed ns2:test-16916488687341,,1691648901068.4897eee4660ec19a1cc212f00aac000c. 2023-08-10T06:28:21,516 DEBUG [RegionOpenAndInit-ns2:test-16916488687341-pool-0 {}] regionserver.HRegion(1622): Region close journal for 4897eee4660ec19a1cc212f00aac000c: 2023-08-10T06:28:21,518 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:test-16916488687341,,1691648901068.4897eee4660ec19a1cc212f00aac000c.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1691648901517"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691648901517"}]},"ts":"1691648901517"} 2023-08-10T06:28:21,520 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:28:21,521 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16916488687341","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648901521"}]},"ts":"1691648901521"} 2023-08-10T06:28:21,522 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16916488687341, state=ENABLING in hbase:meta 2023-08-10T06:28:21,527 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=42, ppid=41, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16916488687341, region=4897eee4660ec19a1cc212f00aac000c, ASSIGN}] 2023-08-10T06:28:21,529 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=42, ppid=41, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16916488687341, region=4897eee4660ec19a1cc212f00aac000c, ASSIGN 2023-08-10T06:28:21,529 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=42, ppid=41, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:test-16916488687341, region=4897eee4660ec19a1cc212f00aac000c, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45101,1691648859755; forceNewPlan=false, retain=false 2023-08-10T06:28:21,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=41 2023-08-10T06:28:21,681 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=42 updating hbase:meta row=4897eee4660ec19a1cc212f00aac000c, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:21,683 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE; OpenRegionProcedure 4897eee4660ec19a1cc212f00aac000c, server=jenkins-hbase3.apache.org,45101,1691648859755}] 2023-08-10T06:28:21,840 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(130): Open ns2:test-16916488687341,,1691648901068.4897eee4660ec19a1cc212f00aac000c. 2023-08-10T06:28:21,841 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7254): Opening region: {ENCODED => 4897eee4660ec19a1cc212f00aac000c, NAME => 'ns2:test-16916488687341,,1691648901068.4897eee4660ec19a1cc212f00aac000c.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:28:21,841 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:28:21,841 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16916488687341 4897eee4660ec19a1cc212f00aac000c 2023-08-10T06:28:21,841 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(888): Instantiated ns2:test-16916488687341,,1691648901068.4897eee4660ec19a1cc212f00aac000c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:28:21,841 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7296): checking encryption for 4897eee4660ec19a1cc212f00aac000c 2023-08-10T06:28:21,841 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7299): checking classloading for 4897eee4660ec19a1cc212f00aac000c 2023-08-10T06:28:21,843 INFO [StoreOpener-4897eee4660ec19a1cc212f00aac000c-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 4897eee4660ec19a1cc212f00aac000c 2023-08-10T06:28:21,845 INFO [StoreOpener-4897eee4660ec19a1cc212f00aac000c-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4897eee4660ec19a1cc212f00aac000c columnFamilyName f 2023-08-10T06:28:21,845 DEBUG [StoreOpener-4897eee4660ec19a1cc212f00aac000c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:28:21,845 INFO [StoreOpener-4897eee4660ec19a1cc212f00aac000c-1 {}] regionserver.HStore(324): Store=4897eee4660ec19a1cc212f00aac000c/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:28:21,846 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/4897eee4660ec19a1cc212f00aac000c 2023-08-10T06:28:21,847 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/4897eee4660ec19a1cc212f00aac000c 2023-08-10T06:28:21,849 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1076): writing seq id for 4897eee4660ec19a1cc212f00aac000c 2023-08-10T06:28:21,852 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/4897eee4660ec19a1cc212f00aac000c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:28:21,852 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): Opened 4897eee4660ec19a1cc212f00aac000c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9949007840, jitterRate=-0.07342644035816193}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-08-10T06:28:21,853 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(992): Region open journal for 4897eee4660ec19a1cc212f00aac000c: 2023-08-10T06:28:21,854 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2552): Post open deploy tasks for ns2:test-16916488687341,,1691648901068.4897eee4660ec19a1cc212f00aac000c., pid=43, masterSystemTime=1691648901835 2023-08-10T06:28:21,857 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2579): Finished post open deploy task for ns2:test-16916488687341,,1691648901068.4897eee4660ec19a1cc212f00aac000c. 2023-08-10T06:28:21,857 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(158): Opened ns2:test-16916488687341,,1691648901068.4897eee4660ec19a1cc212f00aac000c. 2023-08-10T06:28:21,857 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=42 updating hbase:meta row=4897eee4660ec19a1cc212f00aac000c, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:21,866 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=43, resume processing ppid=42 2023-08-10T06:28:21,867 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=43, ppid=42, state=SUCCESS; OpenRegionProcedure 4897eee4660ec19a1cc212f00aac000c, server=jenkins-hbase3.apache.org,45101,1691648859755 in 176 msec 2023-08-10T06:28:21,869 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=42, resume processing ppid=41 2023-08-10T06:28:21,869 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=42, ppid=41, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-16916488687341, region=4897eee4660ec19a1cc212f00aac000c, ASSIGN in 340 msec 2023-08-10T06:28:21,869 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16916488687341","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648901869"}]},"ts":"1691648901869"} 2023-08-10T06:28:21,872 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16916488687341, state=ENABLED in hbase:meta 2023-08-10T06:28:21,874 DEBUG [PEWorker-3 {}] procedure.TruncateTableProcedure(145): truncate 'ns2:test-16916488687341' completed 2023-08-10T06:28:21,875 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=41, state=SUCCESS; TruncateTableProcedure (table=ns2:test-16916488687341 preserveSplits=true) in 848 msec 2023-08-10T06:28:22,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=41 2023-08-10T06:28:22,145 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] client.HBaseAdmin$TableFuture(3590): Operation: TRUNCATE, Table Name: ns2:test-16916488687341, procId: 41 completed 2023-08-10T06:28:22,154 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:45379/backupUT/backup_1691648877989/ns2/test-16916488687341/archive/data/ns2/test-16916488687341/455351a9679f34a345fb40a29340fd37 from hbase tables ns2:test-16916488687341 to tables ns2:test-16916488687341 2023-08-10T06:28:22,154 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceRestoreJob(72): Restore ns2:test-16916488687341 into ns2:test-16916488687341 2023-08-10T06:28:22,157 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-ns2-test-16916488687341-1691648902154 from hdfs://localhost:45379/backupUT/backup_1691648877989/ns2/test-16916488687341/archive/data/ns2/test-16916488687341/455351a9679f34a345fb40a29340fd37 2023-08-10T06:28:22,160 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x01611efc to 127.0.0.1:61704 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:28:22,169 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32fe2db7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:28:22,171 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:28:22,173 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:39120, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:28:22,189 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-08-10T06:28:22,189 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:test-16916488687341 2023-08-10T06:28:22,194 DEBUG [hconnection-0x20fa3771-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:28:22,196 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:38182, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:28:22,199 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:test-16916488687341: 2023-08-10T06:28:22,200 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-08-10T06:28:22,200 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/partitions_b15d1624-a04e-4f82-8575-2dd4c7d6dc65 2023-08-10T06:28:22,645 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:22,646 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:22,646 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:22,647 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:23,682 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/hadoop-7333913883544945875.jar 2023-08-10T06:28:23,682 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:23,683 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:23,683 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:23,684 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:23,684 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:23,685 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:23,685 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:23,685 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:23,686 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:23,687 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:28:23,687 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-08-10T06:28:23,688 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-08-10T06:28:23,688 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-08-10T06:28:23,689 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-08-10T06:28:23,689 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-08-10T06:28:23,689 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-08-10T06:28:23,690 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-08-10T06:28:23,690 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-08-10T06:28:23,691 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-08-10T06:28:23,691 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-08-10T06:28:23,692 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-08-10T06:28:23,695 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:23,696 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:23,696 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:23,696 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:23,697 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:23,697 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:23,698 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:28:23,698 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: ns2:test-16916488687341 2023-08-10T06:28:23,698 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] client.ConnectionImplementation(2114): Closing master protocol: MasterService 2023-08-10T06:28:23,698 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x01611efc to 127.0.0.1:61704 2023-08-10T06:28:23,698 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:28:23,699 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-08-10T06:28:23,699 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-08-10T06:28:23,699 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:28:23,705 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-08-10T06:28:23,711 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-08-10T06:28:23,806 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691648903763/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-08-10T06:28:23,806 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691648903763/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-08-10T06:28:23,910 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:45379/backupUT/backup_1691648877989/ns2/test-16916488687341/archive/data/ns2/test-16916488687341/455351a9679f34a345fb40a29340fd37/f/7dc1c6bf67474a4488e888a4dcbb27b6 2023-08-10T06:28:23,916 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-08-10T06:28:23,921 DEBUG [LocalJobRunner Map Task Executor #0 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 7dc1c6bf67474a4488e888a4dcbb27b6 2023-08-10T06:28:24,053 DEBUG [pool-200-thread-1 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x433b71ab to 127.0.0.1:61704 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:28:24,059 DEBUG [pool-200-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@458547b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:28:24,062 DEBUG [hconnection-0x852b684-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:28:24,063 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:38194, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:28:24,068 DEBUG [pool-200-thread-1 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x433b71ab to 127.0.0.1:61704 2023-08-10T06:28:24,069 DEBUG [pool-200-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:28:24,069 DEBUG [pool-200-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-08-10T06:28:24,069 DEBUG [pool-200-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-08-10T06:28:24,086 DEBUG [pool-200-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-ns2-test-16916488687341-1691648902154/_temporary/0/_temporary/attempt_local2000639069_0003_r_000000_0/f/5920776c77b04836a3ebc66cf5b0d935 is 37, key is row10/f:q1/1691648873613/Put/seqid=0 2023-08-10T06:28:24,088 WARN [Thread-821 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:45101] Specified favored nodes: [jenkins-hbase3.apache.org:45101] 2023-08-10T06:28:24,868 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-ns2-test-16916488687341-1691648902154 2023-08-10T06:28:24,869 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x458a42a0 to 127.0.0.1:61704 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:28:24,874 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2023-08-10T06:28:24,874 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2023-08-10T06:28:24,875 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@429ae52b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:28:24,877 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:28:24,879 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:38198, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:28:24,888 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-ns2-test-16916488687341-1691648902154/_SUCCESS 2023-08-10T06:28:24,894 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:28:24,895 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:39136, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:28:24,927 DEBUG [LoadIncrementalHFiles-0 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 5920776c77b04836a3ebc66cf5b0d935 2023-08-10T06:28:24,928 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-ns2-test-16916488687341-1691648902154/f/5920776c77b04836a3ebc66cf5b0d935 first=Optional[row0] last=Optional[row98] 2023-08-10T06:28:24,947 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=ns2:test-16916488687341,,1691648901068.4897eee4660ec19a1cc212f00aac000c., hostname=jenkins-hbase3.apache.org,45101,1691648859755, seqNum=2 for row with hfile group [{f,hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-ns2-test-16916488687341-1691648902154/f/5920776c77b04836a3ebc66cf5b0d935}] 2023-08-10T06:28:24,968 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-ns2-test-16916488687341-1691648902154/f/5920776c77b04836a3ebc66cf5b0d935 for inclusion in 4897eee4660ec19a1cc212f00aac000c/f 2023-08-10T06:28:24,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-08-10T06:28:24,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HStore(625): Region bounds: first= last= 2023-08-10T06:28:24,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 5920776c77b04836a3ebc66cf5b0d935 2023-08-10T06:28:24,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HRegion(2520): Flush status journal for 4897eee4660ec19a1cc212f00aac000c: 2023-08-10T06:28:24,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-ns2-test-16916488687341-1691648902154/f/5920776c77b04836a3ebc66cf5b0d935 to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/staging/jenkins__ns2_test-16916488687341__2svi0i7ee0tf5rev37895af903nc4hvjd1ba5k3udaq4krmpglkhlj1hf73nsoq/f/5920776c77b04836a3ebc66cf5b0d935 2023-08-10T06:28:24,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/staging/jenkins__ns2_test-16916488687341__2svi0i7ee0tf5rev37895af903nc4hvjd1ba5k3udaq4krmpglkhlj1hf73nsoq/f/5920776c77b04836a3ebc66cf5b0d935 as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/4897eee4660ec19a1cc212f00aac000c/f/a272e7ab9ac64bbb812d9b65df841988_SeqId_4_ 2023-08-10T06:28:24,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x50d2d76b to 127.0.0.1:61704 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:28:24,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27f394cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:28:24,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:28:24,992 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:39142, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:28:24,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:28:24,998 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:38206, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:28:25,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] impl.BackupSystemTable(408): write bulk load descriptor to backup ns2:test-16916488687341 with 1 entries 2023-08-10T06:28:25,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] impl.BackupSystemTable(1582): writing raw bulk path hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/4897eee4660ec19a1cc212f00aac000c/f/a272e7ab9ac64bbb812d9b65df841988_SeqId_4_ for ns2:test-16916488687341 4897eee4660ec19a1cc212f00aac000c 2023-08-10T06:28:25,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] impl.BackupSystemTable(415): written 1 rows for bulk load of ns2:test-16916488687341 2023-08-10T06:28:25,023 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] client.ConnectionImplementation(2114): Closing master protocol: MasterService 2023-08-10T06:28:25,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x50d2d76b to 127.0.0.1:61704 2023-08-10T06:28:25,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:28:25,025 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/staging/jenkins__ns2_test-16916488687341__2svi0i7ee0tf5rev37895af903nc4hvjd1ba5k3udaq4krmpglkhlj1hf73nsoq/f/5920776c77b04836a3ebc66cf5b0d935 into 4897eee4660ec19a1cc212f00aac000c/f as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/4897eee4660ec19a1cc212f00aac000c/f/a272e7ab9ac64bbb812d9b65df841988_SeqId_4_ - updating store file list. 2023-08-10T06:28:25,035 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/4897eee4660ec19a1cc212f00aac000c/f/a272e7ab9ac64bbb812d9b65df841988_SeqId_4_ into 4897eee4660ec19a1cc212f00aac000c/f 2023-08-10T06:28:25,035 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/staging/jenkins__ns2_test-16916488687341__2svi0i7ee0tf5rev37895af903nc4hvjd1ba5k3udaq4krmpglkhlj1hf73nsoq/f/5920776c77b04836a3ebc66cf5b0d935 into 4897eee4660ec19a1cc212f00aac000c/f (new location: hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/4897eee4660ec19a1cc212f00aac000c/f/a272e7ab9ac64bbb812d9b65df841988_SeqId_4_) 2023-08-10T06:28:25,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/staging/jenkins__ns2_test-16916488687341__2svi0i7ee0tf5rev37895af903nc4hvjd1ba5k3udaq4krmpglkhlj1hf73nsoq/f/5920776c77b04836a3ebc66cf5b0d935 2023-08-10T06:28:25,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1115fcfd to 127.0.0.1:61704 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:28:25,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a5ab7cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:28:25,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:28:25,057 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:39150, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-08-10T06:28:25,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:28:25,062 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:38208, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-08-10T06:28:25,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] impl.BackupSystemTable(388): write bulk load descriptor to backup ns2:test-16916488687341 with 1 entries 2023-08-10T06:28:25,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] impl.BackupSystemTable(1503): writing done bulk path hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/4897eee4660ec19a1cc212f00aac000c/f/a272e7ab9ac64bbb812d9b65df841988_SeqId_4_ for ns2:test-16916488687341 4897eee4660ec19a1cc212f00aac000c 2023-08-10T06:28:25,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] impl.BackupSystemTable(394): written 1 rows for bulk load of ns2:test-16916488687341 2023-08-10T06:28:25,082 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] client.ConnectionImplementation(2114): Closing master protocol: MasterService 2023-08-10T06:28:25,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1115fcfd to 127.0.0.1:61704 2023-08-10T06:28:25,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:28:25,095 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] client.ConnectionImplementation(2114): Closing master protocol: MasterService 2023-08-10T06:28:25,095 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x458a42a0 to 127.0.0.1:61704 2023-08-10T06:28:25,095 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:28:25,096 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-08-10T06:28:25,096 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-08-10T06:28:25,096 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(233): Backup: backup_1691648877989 hdfs://localhost:45379/backupUT/backup_1691648877989/ns2/test-16916488687341/ 2023-08-10T06:28:25,097 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:45379/backupUT/backup_1691648877989/.backup.manifest 2023-08-10T06:28:25,100 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691648877989/.backup.manifest 2023-08-10T06:28:25,100 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(148): Restoring 'ns3:test-16916488687342' to 'ns3:test-16916488687342' from full backup image hdfs://localhost:45379/backupUT/backup_1691648877989/ns3/test-16916488687342 2023-08-10T06:28:25,106 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] util.RestoreTool(101): Folder tableArchivePath: hdfs://localhost:45379/backupUT/backup_1691648877989/ns3/test-16916488687342/archive/data/ns3/test-16916488687342 does not exists 2023-08-10T06:28:25,107 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] util.RestoreTool(326): find table descriptor but no archive dir for table ns3:test-16916488687342, will only create table 2023-08-10T06:28:25,108 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] util.RestoreTool(476): Truncating exising target table 'ns3:test-16916488687342', preserving region splits 2023-08-10T06:28:25,109 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] client.HBaseAdmin$18(963): Started disable of ns3:test-16916488687342 2023-08-10T06:28:25,109 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster$12(2663): Client=jenkins//172.31.12.81 disable ns3:test-16916488687342 2023-08-10T06:28:25,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=44, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=ns3:test-16916488687342 2023-08-10T06:28:25,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=44 2023-08-10T06:28:25,114 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16916488687342","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648905114"}]},"ts":"1691648905114"} 2023-08-10T06:28:25,116 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16916488687342, state=DISABLING in hbase:meta 2023-08-10T06:28:25,117 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(293): Set ns3:test-16916488687342 to state=DISABLING 2023-08-10T06:28:25,118 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns3:test-16916488687342, region=9830345ee6e7e7f9278f341f1011836d, UNASSIGN}] 2023-08-10T06:28:25,120 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=45, ppid=44, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns3:test-16916488687342, region=9830345ee6e7e7f9278f341f1011836d, UNASSIGN 2023-08-10T06:28:25,121 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=45 updating hbase:meta row=9830345ee6e7e7f9278f341f1011836d, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:25,123 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE; CloseRegionProcedure 9830345ee6e7e7f9278f341f1011836d, server=jenkins-hbase3.apache.org,45101,1691648859755}] 2023-08-10T06:28:25,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=44 2023-08-10T06:28:25,275 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] handler.UnassignRegionHandler(119): Close 9830345ee6e7e7f9278f341f1011836d 2023-08-10T06:28:25,275 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1668): Closing 9830345ee6e7e7f9278f341f1011836d, disabling compactions & flushes 2023-08-10T06:28:25,275 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1690): Closing region ns3:test-16916488687342,,1691648873695.9830345ee6e7e7f9278f341f1011836d. 2023-08-10T06:28:25,275 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16916488687342,,1691648873695.9830345ee6e7e7f9278f341f1011836d. 2023-08-10T06:28:25,275 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16916488687342,,1691648873695.9830345ee6e7e7f9278f341f1011836d. after waiting 0 ms 2023-08-10T06:28:25,275 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16916488687342,,1691648873695.9830345ee6e7e7f9278f341f1011836d. 2023-08-10T06:28:25,280 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns3/test-16916488687342/9830345ee6e7e7f9278f341f1011836d/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2023-08-10T06:28:25,280 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:28:25,281 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1909): Closed ns3:test-16916488687342,,1691648873695.9830345ee6e7e7f9278f341f1011836d. 2023-08-10T06:28:25,282 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1622): Region close journal for 9830345ee6e7e7f9278f341f1011836d: 2023-08-10T06:28:25,283 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] handler.UnassignRegionHandler(163): Closed 9830345ee6e7e7f9278f341f1011836d 2023-08-10T06:28:25,284 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=45 updating hbase:meta row=9830345ee6e7e7f9278f341f1011836d, regionState=CLOSED 2023-08-10T06:28:25,287 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=46, resume processing ppid=45 2023-08-10T06:28:25,288 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=46, ppid=45, state=SUCCESS; CloseRegionProcedure 9830345ee6e7e7f9278f341f1011836d, server=jenkins-hbase3.apache.org,45101,1691648859755 in 164 msec 2023-08-10T06:28:25,289 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=45, resume processing ppid=44 2023-08-10T06:28:25,289 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=45, ppid=44, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-16916488687342, region=9830345ee6e7e7f9278f341f1011836d, UNASSIGN in 170 msec 2023-08-10T06:28:25,290 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16916488687342","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648905290"}]},"ts":"1691648905290"} 2023-08-10T06:28:25,291 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16916488687342, state=DISABLED in hbase:meta 2023-08-10T06:28:25,294 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(305): Set ns3:test-16916488687342 to state=DISABLED 2023-08-10T06:28:25,296 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=44, state=SUCCESS; DisableTableProcedure table=ns3:test-16916488687342 in 186 msec 2023-08-10T06:28:25,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=44 2023-08-10T06:28:25,417 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] client.HBaseAdmin$TableFuture(3590): Operation: DISABLE, Table Name: ns3:test-16916488687342, procId: 44 completed 2023-08-10T06:28:25,417 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] client.HBaseAdmin$16(843): Started truncating ns3:test-16916488687342 2023-08-10T06:28:25,418 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster$6(2479): Client=jenkins//172.31.12.81 truncate ns3:test-16916488687342 2023-08-10T06:28:25,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=47, state=RUNNABLE:TRUNCATE_TABLE_PRE_OPERATION; TruncateTableProcedure (table=ns3:test-16916488687342 preserveSplits=true) 2023-08-10T06:28:25,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=47 2023-08-10T06:28:25,421 DEBUG [PEWorker-5 {}] procedure.TruncateTableProcedure(87): waiting for 'ns3:test-16916488687342' regions in transition 2023-08-10T06:28:25,424 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns3/test-16916488687342/9830345ee6e7e7f9278f341f1011836d 2023-08-10T06:28:25,426 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns3/test-16916488687342/9830345ee6e7e7f9278f341f1011836d/f, FileablePath, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns3/test-16916488687342/9830345ee6e7e7f9278f341f1011836d/recovered.edits] 2023-08-10T06:28:25,433 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns3/test-16916488687342/9830345ee6e7e7f9278f341f1011836d/recovered.edits/5.seqid to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/archive/data/ns3/test-16916488687342/9830345ee6e7e7f9278f341f1011836d/recovered.edits/5.seqid 2023-08-10T06:28:25,434 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns3/test-16916488687342/9830345ee6e7e7f9278f341f1011836d 2023-08-10T06:28:25,434 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(293): Archived ns3:test-16916488687342 regions 2023-08-10T06:28:25,440 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of ns3:test-16916488687342 from hbase:meta 2023-08-10T06:28:25,442 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(387): Removing 'ns3:test-16916488687342' descriptor. 2023-08-10T06:28:25,442 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(377): Removing 'ns3:test-16916488687342' from region states. 2023-08-10T06:28:25,442 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns3:test-16916488687342,,1691648873695.9830345ee6e7e7f9278f341f1011836d.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1691648905442"}]},"ts":"9223372036854775807"} 2023-08-10T06:28:25,444 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-08-10T06:28:25,444 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 9830345ee6e7e7f9278f341f1011836d, NAME => 'ns3:test-16916488687342,,1691648873695.9830345ee6e7e7f9278f341f1011836d.', STARTKEY => '', ENDKEY => ''}] 2023-08-10T06:28:25,444 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(381): Marking 'ns3:test-16916488687342' as deleted. 2023-08-10T06:28:25,445 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns3:test-16916488687342","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1691648905444"}]},"ts":"9223372036854775807"} 2023-08-10T06:28:25,447 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1678): Deleted table ns3:test-16916488687342 state from META 2023-08-10T06:28:25,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=47 2023-08-10T06:28:25,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=47 2023-08-10T06:28:25,857 INFO [RegionOpenAndInit-ns3:test-16916488687342-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 77e10d7389ad2c7a8dfcbed4a1e8d4af, NAME => 'ns3:test-16916488687342,,1691648905436.77e10d7389ad2c7a8dfcbed4a1e8d4af.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-16916488687342', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e 2023-08-10T06:28:26,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=47 2023-08-10T06:28:26,264 DEBUG [RegionOpenAndInit-ns3:test-16916488687342-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:test-16916488687342,,1691648905436.77e10d7389ad2c7a8dfcbed4a1e8d4af.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:28:26,264 DEBUG [RegionOpenAndInit-ns3:test-16916488687342-pool-0 {}] regionserver.HRegion(1668): Closing 77e10d7389ad2c7a8dfcbed4a1e8d4af, disabling compactions & flushes 2023-08-10T06:28:26,264 INFO [RegionOpenAndInit-ns3:test-16916488687342-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:test-16916488687342,,1691648905436.77e10d7389ad2c7a8dfcbed4a1e8d4af. 2023-08-10T06:28:26,264 DEBUG [RegionOpenAndInit-ns3:test-16916488687342-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16916488687342,,1691648905436.77e10d7389ad2c7a8dfcbed4a1e8d4af. 2023-08-10T06:28:26,264 DEBUG [RegionOpenAndInit-ns3:test-16916488687342-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16916488687342,,1691648905436.77e10d7389ad2c7a8dfcbed4a1e8d4af. after waiting 0 ms 2023-08-10T06:28:26,264 DEBUG [RegionOpenAndInit-ns3:test-16916488687342-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16916488687342,,1691648905436.77e10d7389ad2c7a8dfcbed4a1e8d4af. 2023-08-10T06:28:26,264 INFO [RegionOpenAndInit-ns3:test-16916488687342-pool-0 {}] regionserver.HRegion(1909): Closed ns3:test-16916488687342,,1691648905436.77e10d7389ad2c7a8dfcbed4a1e8d4af. 2023-08-10T06:28:26,264 DEBUG [RegionOpenAndInit-ns3:test-16916488687342-pool-0 {}] regionserver.HRegion(1622): Region close journal for 77e10d7389ad2c7a8dfcbed4a1e8d4af: 2023-08-10T06:28:26,266 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:test-16916488687342,,1691648905436.77e10d7389ad2c7a8dfcbed4a1e8d4af.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1691648906266"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691648906266"}]},"ts":"1691648906266"} 2023-08-10T06:28:26,267 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:28:26,269 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16916488687342","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648906268"}]},"ts":"1691648906268"} 2023-08-10T06:28:26,270 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16916488687342, state=ENABLING in hbase:meta 2023-08-10T06:28:26,274 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=48, ppid=47, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16916488687342, region=77e10d7389ad2c7a8dfcbed4a1e8d4af, ASSIGN}] 2023-08-10T06:28:26,276 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=48, ppid=47, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16916488687342, region=77e10d7389ad2c7a8dfcbed4a1e8d4af, ASSIGN 2023-08-10T06:28:26,277 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=48, ppid=47, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:test-16916488687342, region=77e10d7389ad2c7a8dfcbed4a1e8d4af, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45101,1691648859755; forceNewPlan=false, retain=false 2023-08-10T06:28:26,428 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=48 updating hbase:meta row=77e10d7389ad2c7a8dfcbed4a1e8d4af, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:26,430 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE; OpenRegionProcedure 77e10d7389ad2c7a8dfcbed4a1e8d4af, server=jenkins-hbase3.apache.org,45101,1691648859755}] 2023-08-10T06:28:26,449 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-08-10T06:28:26,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=47 2023-08-10T06:28:26,586 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(130): Open ns3:test-16916488687342,,1691648905436.77e10d7389ad2c7a8dfcbed4a1e8d4af. 2023-08-10T06:28:26,586 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7254): Opening region: {ENCODED => 77e10d7389ad2c7a8dfcbed4a1e8d4af, NAME => 'ns3:test-16916488687342,,1691648905436.77e10d7389ad2c7a8dfcbed4a1e8d4af.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:28:26,586 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:28:26,586 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16916488687342 77e10d7389ad2c7a8dfcbed4a1e8d4af 2023-08-10T06:28:26,586 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(888): Instantiated ns3:test-16916488687342,,1691648905436.77e10d7389ad2c7a8dfcbed4a1e8d4af.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:28:26,587 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7296): checking encryption for 77e10d7389ad2c7a8dfcbed4a1e8d4af 2023-08-10T06:28:26,587 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7299): checking classloading for 77e10d7389ad2c7a8dfcbed4a1e8d4af 2023-08-10T06:28:26,588 INFO [StoreOpener-77e10d7389ad2c7a8dfcbed4a1e8d4af-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 77e10d7389ad2c7a8dfcbed4a1e8d4af 2023-08-10T06:28:26,590 INFO [StoreOpener-77e10d7389ad2c7a8dfcbed4a1e8d4af-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 77e10d7389ad2c7a8dfcbed4a1e8d4af columnFamilyName f 2023-08-10T06:28:26,590 DEBUG [StoreOpener-77e10d7389ad2c7a8dfcbed4a1e8d4af-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:28:26,590 INFO [StoreOpener-77e10d7389ad2c7a8dfcbed4a1e8d4af-1 {}] regionserver.HStore(324): Store=77e10d7389ad2c7a8dfcbed4a1e8d4af/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:28:26,591 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns3/test-16916488687342/77e10d7389ad2c7a8dfcbed4a1e8d4af 2023-08-10T06:28:26,592 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns3/test-16916488687342/77e10d7389ad2c7a8dfcbed4a1e8d4af 2023-08-10T06:28:26,594 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1076): writing seq id for 77e10d7389ad2c7a8dfcbed4a1e8d4af 2023-08-10T06:28:26,597 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns3/test-16916488687342/77e10d7389ad2c7a8dfcbed4a1e8d4af/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:28:26,597 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): Opened 77e10d7389ad2c7a8dfcbed4a1e8d4af; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9888468640, jitterRate=-0.07906459271907806}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-08-10T06:28:26,598 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(992): Region open journal for 77e10d7389ad2c7a8dfcbed4a1e8d4af: 2023-08-10T06:28:26,599 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2552): Post open deploy tasks for ns3:test-16916488687342,,1691648905436.77e10d7389ad2c7a8dfcbed4a1e8d4af., pid=49, masterSystemTime=1691648906582 2023-08-10T06:28:26,600 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2579): Finished post open deploy task for ns3:test-16916488687342,,1691648905436.77e10d7389ad2c7a8dfcbed4a1e8d4af. 2023-08-10T06:28:26,600 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(158): Opened ns3:test-16916488687342,,1691648905436.77e10d7389ad2c7a8dfcbed4a1e8d4af. 2023-08-10T06:28:26,601 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=48 updating hbase:meta row=77e10d7389ad2c7a8dfcbed4a1e8d4af, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:26,604 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=49, resume processing ppid=48 2023-08-10T06:28:26,604 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=49, ppid=48, state=SUCCESS; OpenRegionProcedure 77e10d7389ad2c7a8dfcbed4a1e8d4af, server=jenkins-hbase3.apache.org,45101,1691648859755 in 172 msec 2023-08-10T06:28:26,606 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=48, resume processing ppid=47 2023-08-10T06:28:26,606 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=48, ppid=47, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-16916488687342, region=77e10d7389ad2c7a8dfcbed4a1e8d4af, ASSIGN in 330 msec 2023-08-10T06:28:26,606 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16916488687342","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648906606"}]},"ts":"1691648906606"} 2023-08-10T06:28:26,607 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16916488687342, state=ENABLED in hbase:meta 2023-08-10T06:28:26,615 DEBUG [PEWorker-1 {}] procedure.TruncateTableProcedure(145): truncate 'ns3:test-16916488687342' completed 2023-08-10T06:28:26,617 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=47, state=SUCCESS; TruncateTableProcedure (table=ns3:test-16916488687342 preserveSplits=true) in 1.1970 sec 2023-08-10T06:28:27,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=47 2023-08-10T06:28:27,529 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] client.HBaseAdmin$TableFuture(3590): Operation: TRUNCATE, Table Name: ns3:test-16916488687342, procId: 47 completed 2023-08-10T06:28:27,533 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-08-10T06:28:27,533 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(233): Backup: backup_1691648877989 hdfs://localhost:45379/backupUT/backup_1691648877989/ns3/test-16916488687342/ 2023-08-10T06:28:27,533 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-08-10T06:28:27,554 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreMultipleOverwrite Thread=597 (was 582) Potentially hanging thread: hconnection-0x69012128-shared-pool-12 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x69012128-shared-pool-13 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x69012128-shared-pool-15 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x69012128-shared-pool-14 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: HFileArchiver-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/cluster_28117de4-e82e-18fe-7324-f4a37a8c1f1e/dfs/data/data2/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-937657374_16 at /127.0.0.1:49738 [Receiving block BP-1233959968-172.31.12.81-1691648853138:blk_1073741857_1033] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: (jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/cluster_28117de4-e82e-18fe-7324-f4a37a8c1f1e/dfs/data/data1/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1703575583_16 at /127.0.0.1:36576 [Waiting for operation #25] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-1233959968-172.31.12.81-1691648853138:blk_1073741857_1033, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-1233959968-172.31.12.81-1691648853138:blk_1073741858_1034, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-937657374_16 at /127.0.0.1:57138 [Waiting for operation #5] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: HFileArchiver-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x69012128-shared-pool-16 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x65702ead-shared-pool-6 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x69012128-shared-pool-11 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-937657374_16 at /127.0.0.1:49742 [Receiving block BP-1233959968-172.31.12.81-1691648853138:blk_1073741858_1034] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=675 (was 660) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=212 (was 208) - SystemLoadAverage LEAK? -, ProcessCount=171 (was 171), AvailableMemoryMB=2471 (was 2147) - AvailableMemoryMB LEAK? - 2023-08-10T06:28:27,554 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(130): Thread=597 is superior to 500 2023-08-10T06:28:27,570 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreSingleOverwrite Thread=597, OpenFileDescriptor=675, MaxFileDescriptor=60000, SystemLoadAverage=212, ProcessCount=171, AvailableMemoryMB=2469 2023-08-10T06:28:27,570 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(130): Thread=597 is superior to 500 2023-08-10T06:28:27,570 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] backup.TestFullRestore(173): test full restore on a single table empty table 2023-08-10T06:28:27,571 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x6f726ee2 to 127.0.0.1:61704 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:28:27,577 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cdf4aea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:28:27,580 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:28:27,584 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:38212, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:28:27,586 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:28:27,587 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:39156, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:28:27,602 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-08-10T06:28:27,609 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:28:27,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691648907609 (current time:1691648907609). 2023-08-10T06:28:27,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:28:27,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-08-10T06:28:27,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0c9ed6c9 to 127.0.0.1:61704 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:28:27,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28c40ff3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:28:27,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:28:27,620 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:38218, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:28:27,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0c9ed6c9 to 127.0.0.1:61704 2023-08-10T06:28:27,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:28:27,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:28:27,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=50, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:28:27,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 50 2023-08-10T06:28:27,625 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:28:27,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=50 2023-08-10T06:28:27,626 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:28:27,628 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:28:27,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=50 2023-08-10T06:28:27,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=50 2023-08-10T06:28:28,035 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:28:28,035 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72}] 2023-08-10T06:28:28,037 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=51, ppid=50, state=RUNNABLE; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:28:28,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45101 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=51 2023-08-10T06:28:28,189 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. 2023-08-10T06:28:28,189 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegion(2819): Flushing b9c493f2c9fbef45de19e8268a534e72 2/2 column families, dataSize=1.51 KB heapSize=2.81 KB 2023-08-10T06:28:28,204 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/meta/bc10682e7f36491d941e2fe62e2e4100 is 171, key is trslm:hdfs://localhost:45379/backupUT\x00ns2:test-16916488687341/meta:log-roll-map/1691648899724/Put/seqid=0 2023-08-10T06:28:28,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=50 2023-08-10T06:28:28,610 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=792 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/meta/bc10682e7f36491d941e2fe62e2e4100 2023-08-10T06:28:28,617 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: bc10682e7f36491d941e2fe62e2e4100 2023-08-10T06:28:28,620 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/session/37561b9a8ae2406bb6d053c33aeff31c is 458, key is session:backup_1691648877989/session:context/1691648900608/Put/seqid=0 2023-08-10T06:28:28,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=50 2023-08-10T06:28:29,025 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=754 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/session/37561b9a8ae2406bb6d053c33aeff31c 2023-08-10T06:28:29,033 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 37561b9a8ae2406bb6d053c33aeff31c 2023-08-10T06:28:29,034 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/meta/bc10682e7f36491d941e2fe62e2e4100 as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/bc10682e7f36491d941e2fe62e2e4100 2023-08-10T06:28:29,042 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/bc10682e7f36491d941e2fe62e2e4100, entries=6, sequenceid=17, filesize=5.8 K 2023-08-10T06:28:29,043 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/session/37561b9a8ae2406bb6d053c33aeff31c as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/37561b9a8ae2406bb6d053c33aeff31c 2023-08-10T06:28:29,050 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/37561b9a8ae2406bb6d053c33aeff31c, entries=2, sequenceid=17, filesize=5.4 K 2023-08-10T06:28:29,051 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegion(3022): Finished flush of dataSize ~1.51 KB/1546, heapSize ~2.78 KB/2848, currentSize=0 B/0 for b9c493f2c9fbef45de19e8268a534e72 in 862ms, sequenceid=17, compaction requested=false 2023-08-10T06:28:29,052 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegion(2520): Flush status journal for b9c493f2c9fbef45de19e8268a534e72: 2023-08-10T06:28:29,052 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. for snapshot_backup_system completed. 2023-08-10T06:28:29,052 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72.' region-info for snapshot=snapshot_backup_system 2023-08-10T06:28:29,052 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:28:29,052 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/bc10682e7f36491d941e2fe62e2e4100] hfiles 2023-08-10T06:28:29,052 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/bc10682e7f36491d941e2fe62e2e4100 for snapshot=snapshot_backup_system 2023-08-10T06:28:29,052 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/e88678d5b4b14eecabac817daf12539e, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/37561b9a8ae2406bb6d053c33aeff31c] hfiles 2023-08-10T06:28:29,052 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/e88678d5b4b14eecabac817daf12539e for snapshot=snapshot_backup_system 2023-08-10T06:28:29,052 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/37561b9a8ae2406bb6d053c33aeff31c for snapshot=snapshot_backup_system 2023-08-10T06:28:29,460 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. 2023-08-10T06:28:29,461 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=51 2023-08-10T06:28:29,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster(3994): Remote procedure done, pid=51 2023-08-10T06:28:29,461 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:28:29,462 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=51, ppid=50, state=RUNNABLE; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:28:29,465 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=51, resume processing ppid=50 2023-08-10T06:28:29,465 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:28:29,465 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=51, ppid=50, state=SUCCESS; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72 in 1.4280 sec 2023-08-10T06:28:29,466 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:28:29,467 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:28:29,467 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-08-10T06:28:29,467 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_backup_system 2023-08-10T06:28:29,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=50 2023-08-10T06:28:29,883 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:28:29,896 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:28:29,897 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_backup_system 2023-08-10T06:28:30,725 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:28:30,725 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 50 2023-08-10T06:28:30,726 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=50, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 3.1030 sec 2023-08-10T06:28:31,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=50 2023-08-10T06:28:31,735 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: backup:system, procId: 50 completed 2023-08-10T06:28:31,735 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(115): Backup backup_1691648907578 started at 1691648911735. 2023-08-10T06:28:31,737 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(119): Backup session backup_1691648907578 has been started. 2023-08-10T06:28:31,742 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-08-10T06:28:31,743 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-08-10T06:28:31,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-08-10T06:28:31,743 INFO [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-08-10T06:28:31,743 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-08-10T06:28:31,744 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-08-10T06:28:31,744 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-08-10T06:28:31,745 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-08-10T06:28:31,745 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:28:31,747 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:31,747 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45101-0x10060ea6e8b0001, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-08-10T06:28:31,747 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:28:31,747 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:28:31,747 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:31,747 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-08-10T06:28:31,747 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-08-10T06:28:31,748 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45101-0x10060ea6e8b0001, quorum=127.0.0.1:61704, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-08-10T06:28:31,748 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-08-10T06:28:31,748 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:28:31,748 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-08-10T06:28:31,748 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-08-10T06:28:31,748 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-08-10T06:28:31,748 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-08-10T06:28:31,748 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-08-10T06:28:31,749 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-08-10T06:28:31,749 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-08-10T06:28:31,749 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,45101,1691648859755' joining acquired barrier for procedure (rolllog) in zk 2023-08-10T06:28:31,753 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:31,753 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-08-10T06:28:31,753 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:31,753 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:28:31,753 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:28:31,753 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45101-0x10060ea6e8b0001, quorum=127.0.0.1:61704, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-08-10T06:28:31,753 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-08-10T06:28:31,753 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:28:31,754 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:28:31,754 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:28:31,755 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:31,755 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:28:31,755 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,45101,1691648859755' joining acquired barrier for procedure 'rolllog' on coordinator 2023-08-10T06:28:31,755 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@27948ad8[Count = 0] remaining members to acquire global barrier 2023-08-10T06:28:31,755 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-08-10T06:28:31,756 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-08-10T06:28:31,757 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45101-0x10060ea6e8b0001, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-08-10T06:28:31,757 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-08-10T06:28:31,757 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-08-10T06:28:31,757 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-08-10T06:28:31,757 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:31,757 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-08-10T06:28:31,757 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-08-10T06:28:31,757 DEBUG [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:31,757 INFO [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1691648885679 highest: 1691648885679 on jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:31,757 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta:.meta(num 1691648885658) roll requested 2023-08-10T06:28:31,770 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691648911758.meta, exclude list is [], retry=0 2023-08-10T06:28:31,775 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK] 2023-08-10T06:28:31,777 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691648885658.meta with entries=24, filesize=6.26 KB; new WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691648911758.meta 2023-08-10T06:28:31,777 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK]] 2023-08-10T06:28:31,777 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691648885658.meta is not closed yet, will try archiving it next time 2023-08-10T06:28:31,777 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C45101%2C1691648859755:(num 1691648885679) roll requested 2023-08-10T06:28:31,780 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691648885658.meta not finished, retry = 0 2023-08-10T06:28:31,790 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691648911777, exclude list is [], retry=0 2023-08-10T06:28:31,794 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK] 2023-08-10T06:28:31,796 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691648885679 with entries=19, filesize=6.26 KB; new WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691648911777 2023-08-10T06:28:31,796 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK]] 2023-08-10T06:28:31,796 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691648885679 is not closed yet, will try archiving it next time 2023-08-10T06:28:31,798 DEBUG [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 41 2023-08-10T06:28:31,798 INFO [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1691648911777 on jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:31,809 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-08-10T06:28:31,809 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,45101,1691648859755' in zk 2023-08-10T06:28:31,813 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-08-10T06:28:31,813 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:31,813 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:28:31,813 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-08-10T06:28:31,813 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:31,813 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:28:31,813 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:28:31,814 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:28:31,815 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:28:31,815 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:28:31,816 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:31,816 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:28:31,817 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:28:31,817 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:31,818 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,45101,1691648859755': 2023-08-10T06:28:31,818 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,45101,1691648859755' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-08-10T06:28:31,818 INFO [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-08-10T06:28:31,818 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-08-10T06:28:31,818 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-08-10T06:28:31,818 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-08-10T06:28:31,818 INFO [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-08-10T06:28:31,821 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-08-10T06:28:31,821 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45101-0x10060ea6e8b0001, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-08-10T06:28:31,821 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45101-0x10060ea6e8b0001, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-08-10T06:28:31,821 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-08-10T06:28:31,821 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-08-10T06:28:31,821 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:28:31,821 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:28:31,821 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:28:31,822 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:28:31,822 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:28:31,822 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:28:31,822 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:31,823 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:28:31,823 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:28:31,823 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:28:31,824 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:28:31,824 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:31,824 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:31,825 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:28:31,825 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:28:31,826 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:31,830 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:31,830 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45101-0x10060ea6e8b0001, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-08-10T06:28:31,830 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45101-0x10060ea6e8b0001, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-08-10T06:28:31,830 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-08-10T06:28:31,831 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:28:31,831 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:28:31,831 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-08-10T06:28:31,831 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:31,831 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-08-10T06:28:31,831 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-08-10T06:28:31,831 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:28:31,831 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-08-10T06:28:31,831 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-08-10T06:28:31,831 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-08-10T06:28:31,831 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:28:31,831 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(347): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Unable to get data of znode /1/rolllog-proc/abort/rolllog because node does not exist (not an error) 2023-08-10T06:28:31,831 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:28:31,832 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] client.HBaseAdmin(2751): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-08-10T06:28:31,832 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] client.HBaseAdmin(2759): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-08-10T06:28:31,932 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] client.HBaseAdmin(2765): Getting current status of procedure from master... 2023-08-10T06:28:31,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-08-10T06:28:31,949 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1691648911948_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } 2023-08-10T06:28:31,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691648911949 (current time:1691648911949). 2023-08-10T06:28:31,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:28:31,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1691648911948_default_test-1691648868734 VERSION not specified, setting to 2 2023-08-10T06:28:31,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x2d79de56 to 127.0.0.1:61704 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:28:31,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25cae79d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:28:31,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:28:31,962 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:47028, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:28:31,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x2d79de56 to 127.0.0.1:61704 2023-08-10T06:28:31,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:28:31,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:28:31,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=52, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1691648911948_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } 2023-08-10T06:28:31,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1691648911948_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 }, snapshot procedure id = 52 2023-08-10T06:28:31,967 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1691648911948_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:28:31,968 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1691648911948_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:28:31,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=52 2023-08-10T06:28:31,970 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1691648911948_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:28:32,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=52 2023-08-10T06:28:32,089 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-08-10T06:28:32,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=52 2023-08-10T06:28:32,389 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1691648911948_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:28:32,389 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE; SnapshotRegionProcedure f320d55a87f9e601302313b6d2cd1a75}] 2023-08-10T06:28:32,391 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=53, ppid=52, state=RUNNABLE; SnapshotRegionProcedure f320d55a87f9e601302313b6d2cd1a75 2023-08-10T06:28:32,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45101 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=53 2023-08-10T06:28:32,543 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1691648868734,,1691648869226.f320d55a87f9e601302313b6d2cd1a75. 2023-08-10T06:28:32,544 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(2819): Flushing f320d55a87f9e601302313b6d2cd1a75 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-08-10T06:28:32,560 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/f320d55a87f9e601302313b6d2cd1a75/.tmp/f/84ce27a7fb634f03ba5a11388462c376 is 37, key is row10/f:q1/1691648871386/Put/seqid=0 2023-08-10T06:28:32,565 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/f320d55a87f9e601302313b6d2cd1a75/.tmp/f/84ce27a7fb634f03ba5a11388462c376 2023-08-10T06:28:32,572 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 84ce27a7fb634f03ba5a11388462c376 2023-08-10T06:28:32,573 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/f320d55a87f9e601302313b6d2cd1a75/.tmp/f/84ce27a7fb634f03ba5a11388462c376 as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/f320d55a87f9e601302313b6d2cd1a75/f/84ce27a7fb634f03ba5a11388462c376 2023-08-10T06:28:32,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=52 2023-08-10T06:28:32,580 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/f320d55a87f9e601302313b6d2cd1a75/f/84ce27a7fb634f03ba5a11388462c376, entries=99, sequenceid=103, filesize=8.2 K 2023-08-10T06:28:32,581 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for f320d55a87f9e601302313b6d2cd1a75 in 37ms, sequenceid=103, compaction requested=false 2023-08-10T06:28:32,583 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(2520): Flush status journal for f320d55a87f9e601302313b6d2cd1a75: 2023-08-10T06:28:32,583 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1691648868734,,1691648869226.f320d55a87f9e601302313b6d2cd1a75. for snapshot_1691648911948_default_test-1691648868734 completed. 2023-08-10T06:28:32,583 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(240): Storing 'test-1691648868734,,1691648869226.f320d55a87f9e601302313b6d2cd1a75.' region-info for snapshot=snapshot_1691648911948_default_test-1691648868734 2023-08-10T06:28:32,583 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:28:32,583 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/f320d55a87f9e601302313b6d2cd1a75/f/84ce27a7fb634f03ba5a11388462c376] hfiles 2023-08-10T06:28:32,583 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/f320d55a87f9e601302313b6d2cd1a75/f/84ce27a7fb634f03ba5a11388462c376 for snapshot=snapshot_1691648911948_default_test-1691648868734 2023-08-10T06:28:32,989 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1691648868734,,1691648869226.f320d55a87f9e601302313b6d2cd1a75. 2023-08-10T06:28:32,989 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=53 2023-08-10T06:28:32,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster(3994): Remote procedure done, pid=53 2023-08-10T06:28:32,990 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1691648911948_default_test-1691648868734 on region f320d55a87f9e601302313b6d2cd1a75 2023-08-10T06:28:32,991 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=53, ppid=52, state=RUNNABLE; SnapshotRegionProcedure f320d55a87f9e601302313b6d2cd1a75 2023-08-10T06:28:32,994 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=53, resume processing ppid=52 2023-08-10T06:28:32,994 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=53, ppid=52, state=SUCCESS; SnapshotRegionProcedure f320d55a87f9e601302313b6d2cd1a75 in 603 msec 2023-08-10T06:28:32,994 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1691648911948_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:28:32,995 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1691648911948_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:28:32,995 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1691648911948_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:28:32,995 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1691648911948_default_test-1691648868734 2023-08-10T06:28:32,996 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_1691648911948_default_test-1691648868734 2023-08-10T06:28:33,010 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1691648911948_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:28:33,016 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1691648911948_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:28:33,017 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_1691648911948_default_test-1691648868734 to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691648911948_default_test-1691648868734 2023-08-10T06:28:33,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=52 2023-08-10T06:28:33,836 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1691648911948_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:28:33,836 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1691648911948_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 }, snapshot procedure id = 52 2023-08-10T06:28:33,837 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=52, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1691648911948_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } in 1.8730 sec 2023-08-10T06:28:34,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=52 2023-08-10T06:28:34,077 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: default:test-1691648868734, procId: 52 completed 2023-08-10T06:28:34,077 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1691648907578 2023-08-10T06:28:34,077 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-08-10T06:28:34,077 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-08-10T06:28:34,077 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1691648907578_test-1691648868734 2023-08-10T06:28:34,077 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1691648911948_default_test-1691648868734 to hdfs://localhost:45379/backupUT/backup_1691648907578/default/test-1691648868734/ 2023-08-10T06:28:34,077 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-08-10T06:28:34,103 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:45379, inputRoot=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e 2023-08-10T06:28:34,103 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,33705,1691648856595_2040486214_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:45379/backupUT/backup_1691648907578/default/test-1691648868734, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:45379/backupUT/backup_1691648907578/default/test-1691648868734/.hbase-snapshot/.tmp/snapshot_1691648911948_default_test-1691648868734 2023-08-10T06:28:34,103 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:45379, inputRoot=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e, snapshotDir=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691648911948_default_test-1691648868734. 2023-08-10T06:28:34,114 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691648911948_default_test-1691648868734 to hdfs://localhost:45379/backupUT/backup_1691648907578/default/test-1691648868734/.hbase-snapshot/.tmp/snapshot_1691648911948_default_test-1691648868734 2023-08-10T06:28:34,135 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:34,136 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:34,137 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:34,137 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:35,139 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/hadoop-1353432792319988172.jar 2023-08-10T06:28:35,140 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:35,140 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:35,141 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:35,141 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:35,142 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:35,142 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:35,143 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:35,143 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:35,144 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:35,145 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:28:35,146 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-08-10T06:28:35,146 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-08-10T06:28:35,147 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-08-10T06:28:35,147 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-08-10T06:28:35,148 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-08-10T06:28:35,148 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-08-10T06:28:35,149 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-08-10T06:28:35,149 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-08-10T06:28:35,149 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-08-10T06:28:35,150 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-08-10T06:28:35,150 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-08-10T06:28:35,151 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:28:35,151 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:28:35,152 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:28:35,153 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:28:35,153 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:28:35,154 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:28:35,155 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:28:35,189 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-08-10T06:28:35,211 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1691648911948_default_test-1691648868734' hfile list 2023-08-10T06:28:35,214 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-08-10T06:28:35,285 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691648915249/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-08-10T06:28:35,285 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691648915249/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-08-10T06:28:35,353 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-08-10T06:28:35,389 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/f320d55a87f9e601302313b6d2cd1a75/f/84ce27a7fb634f03ba5a11388462c376 output=hdfs://localhost:45379/backupUT/backup_1691648907578/default/test-1691648868734/archive/data/default/test-1691648868734/f320d55a87f9e601302313b6d2cd1a75/f/84ce27a7fb634f03ba5a11388462c376 2023-08-10T06:28:35,389 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 4.004M/sec 2023-08-10T06:28:36,334 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-08-10T06:28:36,335 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-08-10T06:28:36,341 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1691648911948_default_test-1691648868734 2023-08-10T06:28:36,341 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1691648911948_default_test-1691648868734 finished. 2023-08-10T06:28:36,342 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(1014): test-1691648868734 2023-08-10T06:28:36,349 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691648907578 2023-08-10T06:28:36,349 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-08-10T06:28:36,756 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:45379/backupUT/backup_1691648907578/.backup.manifest 2023-08-10T06:28:36,756 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(358): Backup backup_1691648907578 finished: type=FULL,tablelist=test-1691648868734,targetRootDir=hdfs://localhost:45379/backupUT,startts=1691648911735,completets=1691648916349,bytescopied=0 2023-08-10T06:28:36,756 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-08-10T06:28:36,756 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1691648911948_default_test-1691648868734 2023-08-10T06:28:36,757 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1691648911948_default_test-1691648868734" 2023-08-10T06:28:36,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1691648911948_default_test-1691648868734 2023-08-10T06:28:36,766 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1691648911948_default_test-1691648868734 for backup backup_1691648907578 succeeded. 2023-08-10T06:28:36,767 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-08-10T06:28:36,773 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-08-10T06:28:36,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-08-10T06:28:36,777 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-08-10T06:28:36,781 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-08-10T06:28:36,788 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(376): Backup backup_1691648907578 completed. 2023-08-10T06:28:36,788 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] client.ConnectionImplementation(2114): Closing master protocol: MasterService 2023-08-10T06:28:36,788 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x6f726ee2 to 127.0.0.1:61704 2023-08-10T06:28:36,788 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:28:36,803 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] backup.TestFullRestore(178): backup complete 2023-08-10T06:28:36,806 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:45379/backupUT/backup_1691648907578/.backup.manifest 2023-08-10T06:28:36,809 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691648907578/.backup.manifest 2023-08-10T06:28:36,810 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:45379/backupUT/backup_1691648907578/.backup.manifest 2023-08-10T06:28:36,812 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691648907578/.backup.manifest 2023-08-10T06:28:36,812 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(148): Restoring 'test-1691648868734' to 'test-1691648868734' from full backup image hdfs://localhost:45379/backupUT/backup_1691648907578/default/test-1691648868734 2023-08-10T06:28:36,821 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] util.RestoreTool(476): Truncating exising target table 'test-1691648868734', preserving region splits 2023-08-10T06:28:36,821 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] client.HBaseAdmin$18(963): Started disable of test-1691648868734 2023-08-10T06:28:36,822 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster$12(2663): Client=jenkins//172.31.12.81 disable test-1691648868734 2023-08-10T06:28:36,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=test-1691648868734 2023-08-10T06:28:36,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=54 2023-08-10T06:28:36,826 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1691648868734","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648916826"}]},"ts":"1691648916826"} 2023-08-10T06:28:36,828 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1691648868734, state=DISABLING in hbase:meta 2023-08-10T06:28:36,829 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(293): Set test-1691648868734 to state=DISABLING 2023-08-10T06:28:36,830 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=test-1691648868734, region=f320d55a87f9e601302313b6d2cd1a75, UNASSIGN}] 2023-08-10T06:28:36,832 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=test-1691648868734, region=f320d55a87f9e601302313b6d2cd1a75, UNASSIGN 2023-08-10T06:28:36,833 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=f320d55a87f9e601302313b6d2cd1a75, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:36,834 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE; CloseRegionProcedure f320d55a87f9e601302313b6d2cd1a75, server=jenkins-hbase3.apache.org,45101,1691648859755}] 2023-08-10T06:28:36,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=54 2023-08-10T06:28:36,986 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] handler.UnassignRegionHandler(119): Close f320d55a87f9e601302313b6d2cd1a75 2023-08-10T06:28:36,986 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1668): Closing f320d55a87f9e601302313b6d2cd1a75, disabling compactions & flushes 2023-08-10T06:28:36,986 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1690): Closing region test-1691648868734,,1691648869226.f320d55a87f9e601302313b6d2cd1a75. 2023-08-10T06:28:36,986 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1691648868734,,1691648869226.f320d55a87f9e601302313b6d2cd1a75. 2023-08-10T06:28:36,987 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1778): Acquired close lock on test-1691648868734,,1691648869226.f320d55a87f9e601302313b6d2cd1a75. after waiting 0 ms 2023-08-10T06:28:36,987 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1788): Updates disabled for region test-1691648868734,,1691648869226.f320d55a87f9e601302313b6d2cd1a75. 2023-08-10T06:28:36,988 DEBUG [StoreFileCloser-f320d55a87f9e601302313b6d2cd1a75-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 84ce27a7fb634f03ba5a11388462c376 2023-08-10T06:28:36,994 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/f320d55a87f9e601302313b6d2cd1a75/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2023-08-10T06:28:36,995 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:28:36,996 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1909): Closed test-1691648868734,,1691648869226.f320d55a87f9e601302313b6d2cd1a75. 2023-08-10T06:28:36,996 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1622): Region close journal for f320d55a87f9e601302313b6d2cd1a75: 2023-08-10T06:28:36,998 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] handler.UnassignRegionHandler(163): Closed f320d55a87f9e601302313b6d2cd1a75 2023-08-10T06:28:36,998 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=f320d55a87f9e601302313b6d2cd1a75, regionState=CLOSED 2023-08-10T06:28:37,002 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=56, resume processing ppid=55 2023-08-10T06:28:37,003 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=56, ppid=55, state=SUCCESS; CloseRegionProcedure f320d55a87f9e601302313b6d2cd1a75, server=jenkins-hbase3.apache.org,45101,1691648859755 in 166 msec 2023-08-10T06:28:37,005 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=55, resume processing ppid=54 2023-08-10T06:28:37,005 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=55, ppid=54, state=SUCCESS; TransitRegionStateProcedure table=test-1691648868734, region=f320d55a87f9e601302313b6d2cd1a75, UNASSIGN in 173 msec 2023-08-10T06:28:37,006 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1691648868734","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648917006"}]},"ts":"1691648917006"} 2023-08-10T06:28:37,007 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1691648868734, state=DISABLED in hbase:meta 2023-08-10T06:28:37,010 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(305): Set test-1691648868734 to state=DISABLED 2023-08-10T06:28:37,016 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=54, state=SUCCESS; DisableTableProcedure table=test-1691648868734 in 192 msec 2023-08-10T06:28:37,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=54 2023-08-10T06:28:37,129 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] client.HBaseAdmin$TableFuture(3590): Operation: DISABLE, Table Name: default:test-1691648868734, procId: 54 completed 2023-08-10T06:28:37,130 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] client.HBaseAdmin$16(843): Started truncating test-1691648868734 2023-08-10T06:28:37,130 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster$6(2479): Client=jenkins//172.31.12.81 truncate test-1691648868734 2023-08-10T06:28:37,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=57, state=RUNNABLE:TRUNCATE_TABLE_PRE_OPERATION; TruncateTableProcedure (table=test-1691648868734 preserveSplits=true) 2023-08-10T06:28:37,133 DEBUG [PEWorker-1 {}] procedure.TruncateTableProcedure(87): waiting for 'test-1691648868734' regions in transition 2023-08-10T06:28:37,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=57 2023-08-10T06:28:37,137 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/f320d55a87f9e601302313b6d2cd1a75 2023-08-10T06:28:37,140 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/f320d55a87f9e601302313b6d2cd1a75/f, FileablePath, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/f320d55a87f9e601302313b6d2cd1a75/recovered.edits] 2023-08-10T06:28:37,145 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/f320d55a87f9e601302313b6d2cd1a75/f/84ce27a7fb634f03ba5a11388462c376 to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/archive/data/default/test-1691648868734/f320d55a87f9e601302313b6d2cd1a75/f/84ce27a7fb634f03ba5a11388462c376 2023-08-10T06:28:37,148 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/f320d55a87f9e601302313b6d2cd1a75/recovered.edits/106.seqid to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/archive/data/default/test-1691648868734/f320d55a87f9e601302313b6d2cd1a75/recovered.edits/106.seqid 2023-08-10T06:28:37,149 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/f320d55a87f9e601302313b6d2cd1a75 2023-08-10T06:28:37,149 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(293): Archived test-1691648868734 regions 2023-08-10T06:28:37,154 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of test-1691648868734 from hbase:meta 2023-08-10T06:28:37,156 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(387): Removing 'test-1691648868734' descriptor. 2023-08-10T06:28:37,157 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(377): Removing 'test-1691648868734' from region states. 2023-08-10T06:28:37,157 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"test-1691648868734,,1691648869226.f320d55a87f9e601302313b6d2cd1a75.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1691648917157"}]},"ts":"9223372036854775807"} 2023-08-10T06:28:37,159 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-08-10T06:28:37,159 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => f320d55a87f9e601302313b6d2cd1a75, NAME => 'test-1691648868734,,1691648869226.f320d55a87f9e601302313b6d2cd1a75.', STARTKEY => '', ENDKEY => ''}] 2023-08-10T06:28:37,159 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(381): Marking 'test-1691648868734' as deleted. 2023-08-10T06:28:37,159 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"test-1691648868734","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1691648917159"}]},"ts":"9223372036854775807"} 2023-08-10T06:28:37,160 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1678): Deleted table test-1691648868734 state from META 2023-08-10T06:28:37,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=57 2023-08-10T06:28:37,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=57 2023-08-10T06:28:37,574 INFO [RegionOpenAndInit-test-1691648868734-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => acd4d10145cd83e849c8f35f15dd289e, NAME => 'test-1691648868734,,1691648917151.acd4d10145cd83e849c8f35f15dd289e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1691648868734', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e 2023-08-10T06:28:37,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=57 2023-08-10T06:28:37,981 DEBUG [RegionOpenAndInit-test-1691648868734-pool-0 {}] regionserver.HRegion(888): Instantiated test-1691648868734,,1691648917151.acd4d10145cd83e849c8f35f15dd289e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:28:37,981 DEBUG [RegionOpenAndInit-test-1691648868734-pool-0 {}] regionserver.HRegion(1668): Closing acd4d10145cd83e849c8f35f15dd289e, disabling compactions & flushes 2023-08-10T06:28:37,981 INFO [RegionOpenAndInit-test-1691648868734-pool-0 {}] regionserver.HRegion(1690): Closing region test-1691648868734,,1691648917151.acd4d10145cd83e849c8f35f15dd289e. 2023-08-10T06:28:37,981 DEBUG [RegionOpenAndInit-test-1691648868734-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1691648868734,,1691648917151.acd4d10145cd83e849c8f35f15dd289e. 2023-08-10T06:28:37,981 DEBUG [RegionOpenAndInit-test-1691648868734-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on test-1691648868734,,1691648917151.acd4d10145cd83e849c8f35f15dd289e. after waiting 0 ms 2023-08-10T06:28:37,981 DEBUG [RegionOpenAndInit-test-1691648868734-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region test-1691648868734,,1691648917151.acd4d10145cd83e849c8f35f15dd289e. 2023-08-10T06:28:37,981 INFO [RegionOpenAndInit-test-1691648868734-pool-0 {}] regionserver.HRegion(1909): Closed test-1691648868734,,1691648917151.acd4d10145cd83e849c8f35f15dd289e. 2023-08-10T06:28:37,981 DEBUG [RegionOpenAndInit-test-1691648868734-pool-0 {}] regionserver.HRegion(1622): Region close journal for acd4d10145cd83e849c8f35f15dd289e: 2023-08-10T06:28:37,983 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"test-1691648868734,,1691648917151.acd4d10145cd83e849c8f35f15dd289e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1691648917982"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691648917982"}]},"ts":"1691648917982"} 2023-08-10T06:28:37,985 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:28:37,986 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1691648868734","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648917986"}]},"ts":"1691648917986"} 2023-08-10T06:28:37,987 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1691648868734, state=ENABLING in hbase:meta 2023-08-10T06:28:37,992 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1691648868734, region=acd4d10145cd83e849c8f35f15dd289e, ASSIGN}] 2023-08-10T06:28:37,994 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1691648868734, region=acd4d10145cd83e849c8f35f15dd289e, ASSIGN 2023-08-10T06:28:37,995 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=test-1691648868734, region=acd4d10145cd83e849c8f35f15dd289e, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45101,1691648859755; forceNewPlan=false, retain=false 2023-08-10T06:28:38,146 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=58 updating hbase:meta row=acd4d10145cd83e849c8f35f15dd289e, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:38,148 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE; OpenRegionProcedure acd4d10145cd83e849c8f35f15dd289e, server=jenkins-hbase3.apache.org,45101,1691648859755}] 2023-08-10T06:28:38,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=57 2023-08-10T06:28:38,303 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(130): Open test-1691648868734,,1691648917151.acd4d10145cd83e849c8f35f15dd289e. 2023-08-10T06:28:38,303 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7254): Opening region: {ENCODED => acd4d10145cd83e849c8f35f15dd289e, NAME => 'test-1691648868734,,1691648917151.acd4d10145cd83e849c8f35f15dd289e.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:28:38,304 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:28:38,304 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1691648868734 acd4d10145cd83e849c8f35f15dd289e 2023-08-10T06:28:38,304 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(888): Instantiated test-1691648868734,,1691648917151.acd4d10145cd83e849c8f35f15dd289e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:28:38,304 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7296): checking encryption for acd4d10145cd83e849c8f35f15dd289e 2023-08-10T06:28:38,304 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7299): checking classloading for acd4d10145cd83e849c8f35f15dd289e 2023-08-10T06:28:38,305 INFO [StoreOpener-acd4d10145cd83e849c8f35f15dd289e-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region acd4d10145cd83e849c8f35f15dd289e 2023-08-10T06:28:38,307 INFO [StoreOpener-acd4d10145cd83e849c8f35f15dd289e-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region acd4d10145cd83e849c8f35f15dd289e columnFamilyName f 2023-08-10T06:28:38,307 DEBUG [StoreOpener-acd4d10145cd83e849c8f35f15dd289e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:28:38,308 INFO [StoreOpener-acd4d10145cd83e849c8f35f15dd289e-1 {}] regionserver.HStore(324): Store=acd4d10145cd83e849c8f35f15dd289e/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:28:38,308 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/acd4d10145cd83e849c8f35f15dd289e 2023-08-10T06:28:38,309 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/acd4d10145cd83e849c8f35f15dd289e 2023-08-10T06:28:38,311 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1076): writing seq id for acd4d10145cd83e849c8f35f15dd289e 2023-08-10T06:28:38,313 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/acd4d10145cd83e849c8f35f15dd289e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:28:38,313 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): Opened acd4d10145cd83e849c8f35f15dd289e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11134845760, jitterRate=0.03701332211494446}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-08-10T06:28:38,313 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(992): Region open journal for acd4d10145cd83e849c8f35f15dd289e: 2023-08-10T06:28:38,314 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2552): Post open deploy tasks for test-1691648868734,,1691648917151.acd4d10145cd83e849c8f35f15dd289e., pid=59, masterSystemTime=1691648918300 2023-08-10T06:28:38,316 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2579): Finished post open deploy task for test-1691648868734,,1691648917151.acd4d10145cd83e849c8f35f15dd289e. 2023-08-10T06:28:38,316 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(158): Opened test-1691648868734,,1691648917151.acd4d10145cd83e849c8f35f15dd289e. 2023-08-10T06:28:38,317 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=58 updating hbase:meta row=acd4d10145cd83e849c8f35f15dd289e, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:38,320 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=59, resume processing ppid=58 2023-08-10T06:28:38,320 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=59, ppid=58, state=SUCCESS; OpenRegionProcedure acd4d10145cd83e849c8f35f15dd289e, server=jenkins-hbase3.apache.org,45101,1691648859755 in 170 msec 2023-08-10T06:28:38,322 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=58, resume processing ppid=57 2023-08-10T06:28:38,322 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=58, ppid=57, state=SUCCESS; TransitRegionStateProcedure table=test-1691648868734, region=acd4d10145cd83e849c8f35f15dd289e, ASSIGN in 328 msec 2023-08-10T06:28:38,322 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1691648868734","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648918322"}]},"ts":"1691648918322"} 2023-08-10T06:28:38,323 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1691648868734, state=ENABLED in hbase:meta 2023-08-10T06:28:38,327 DEBUG [PEWorker-2 {}] procedure.TruncateTableProcedure(145): truncate 'test-1691648868734' completed 2023-08-10T06:28:38,328 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=57, state=SUCCESS; TruncateTableProcedure (table=test-1691648868734 preserveSplits=true) in 1.1970 sec 2023-08-10T06:28:39,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=57 2023-08-10T06:28:39,241 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] client.HBaseAdmin$TableFuture(3590): Operation: TRUNCATE, Table Name: default:test-1691648868734, procId: 57 completed 2023-08-10T06:28:39,245 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:45379/backupUT/backup_1691648907578/default/test-1691648868734/archive/data/default/test-1691648868734/f320d55a87f9e601302313b6d2cd1a75 from hbase tables test-1691648868734 to tables test-1691648868734 2023-08-10T06:28:39,245 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceRestoreJob(72): Restore test-1691648868734 into test-1691648868734 2023-08-10T06:28:39,247 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-default-test-1691648868734-1691648919245 from hdfs://localhost:45379/backupUT/backup_1691648907578/default/test-1691648868734/archive/data/default/test-1691648868734/f320d55a87f9e601302313b6d2cd1a75 2023-08-10T06:28:39,248 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x70f3de49 to 127.0.0.1:61704 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:28:39,253 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@222c6fe6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:28:39,254 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:28:39,256 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:47310, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:28:39,257 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-08-10T06:28:39,257 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table test-1691648868734 2023-08-10T06:28:39,260 DEBUG [hconnection-0x2a48e34c-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:28:39,262 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:47034, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:28:39,263 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for test-1691648868734: 2023-08-10T06:28:39,263 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-08-10T06:28:39,263 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/partitions_f338b8dc-6aba-4414-8b2a-8dc2d154d1fd 2023-08-10T06:28:39,670 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:39,670 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:39,671 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:39,671 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:40,636 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/hadoop-2857518031087467233.jar 2023-08-10T06:28:40,636 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:40,637 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:40,637 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:40,637 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:40,638 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:40,638 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:40,639 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:40,639 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:40,639 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:40,640 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:28:40,641 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-08-10T06:28:40,641 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-08-10T06:28:40,642 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-08-10T06:28:40,642 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-08-10T06:28:40,643 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-08-10T06:28:40,643 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-08-10T06:28:40,643 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-08-10T06:28:40,644 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-08-10T06:28:40,644 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-08-10T06:28:40,645 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-08-10T06:28:40,645 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-08-10T06:28:40,645 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:40,646 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:40,646 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:40,647 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:40,647 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:40,647 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:40,648 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:28:40,648 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: test-1691648868734 2023-08-10T06:28:40,648 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] client.ConnectionImplementation(2114): Closing master protocol: MasterService 2023-08-10T06:28:40,648 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x70f3de49 to 127.0.0.1:61704 2023-08-10T06:28:40,648 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:28:40,648 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-08-10T06:28:40,648 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-08-10T06:28:40,649 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:28:40,654 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-08-10T06:28:40,660 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-08-10T06:28:40,758 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691648920719/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-08-10T06:28:40,758 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691648920719/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-08-10T06:28:40,854 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:45379/backupUT/backup_1691648907578/default/test-1691648868734/archive/data/default/test-1691648868734/f320d55a87f9e601302313b6d2cd1a75/f/84ce27a7fb634f03ba5a11388462c376 2023-08-10T06:28:40,859 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-08-10T06:28:40,861 DEBUG [LocalJobRunner Map Task Executor #0 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 84ce27a7fb634f03ba5a11388462c376 2023-08-10T06:28:40,876 DEBUG [pool-234-thread-1 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5dd845da to 127.0.0.1:61704 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:28:40,881 DEBUG [pool-234-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15768309, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:28:40,882 DEBUG [hconnection-0xc515747-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:28:40,884 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:38850, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:28:40,888 DEBUG [pool-234-thread-1 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5dd845da to 127.0.0.1:61704 2023-08-10T06:28:40,888 DEBUG [pool-234-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:28:40,888 DEBUG [pool-234-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-08-10T06:28:40,888 DEBUG [pool-234-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-08-10T06:28:40,896 DEBUG [pool-234-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-default-test-1691648868734-1691648919245/_temporary/0/_temporary/attempt_local255450903_0005_r_000000_0/f/4331be33de5e43ab8d334bb624764b87 is 37, key is row10/f:q1/1691648871386/Put/seqid=0 2023-08-10T06:28:40,898 WARN [Thread-992 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:45101] Specified favored nodes: [jenkins-hbase3.apache.org:45101] 2023-08-10T06:28:41,853 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-default-test-1691648868734-1691648919245 2023-08-10T06:28:41,853 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0af77597 to 127.0.0.1:61704 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:28:41,862 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@421ceec7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:28:41,864 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:28:41,865 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:38866, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:28:41,873 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-default-test-1691648868734-1691648919245/_SUCCESS 2023-08-10T06:28:41,878 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:28:41,879 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:48450, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:28:41,891 DEBUG [LoadIncrementalHFiles-0 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 4331be33de5e43ab8d334bb624764b87 2023-08-10T06:28:41,891 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-default-test-1691648868734-1691648919245/f/4331be33de5e43ab8d334bb624764b87 first=Optional[row0] last=Optional[row98] 2023-08-10T06:28:41,893 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=test-1691648868734,,1691648917151.acd4d10145cd83e849c8f35f15dd289e., hostname=jenkins-hbase3.apache.org,45101,1691648859755, seqNum=2 for row with hfile group [{f,hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-default-test-1691648868734-1691648919245/f/4331be33de5e43ab8d334bb624764b87}] 2023-08-10T06:28:41,899 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-default-test-1691648868734-1691648919245/f/4331be33de5e43ab8d334bb624764b87 for inclusion in acd4d10145cd83e849c8f35f15dd289e/f 2023-08-10T06:28:41,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-08-10T06:28:41,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HStore(625): Region bounds: first= last= 2023-08-10T06:28:41,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 4331be33de5e43ab8d334bb624764b87 2023-08-10T06:28:41,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HRegion(2520): Flush status journal for acd4d10145cd83e849c8f35f15dd289e: 2023-08-10T06:28:41,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-default-test-1691648868734-1691648919245/f/4331be33de5e43ab8d334bb624764b87 to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/staging/jenkins__test-1691648868734__7thi6i6edovl5uuqodr6v7roa5899ajf1rc4p1h6cm4t6og2pes8c25gsufbnp22/f/4331be33de5e43ab8d334bb624764b87 2023-08-10T06:28:41,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/staging/jenkins__test-1691648868734__7thi6i6edovl5uuqodr6v7roa5899ajf1rc4p1h6cm4t6og2pes8c25gsufbnp22/f/4331be33de5e43ab8d334bb624764b87 as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/acd4d10145cd83e849c8f35f15dd289e/f/eb3ec537e8f043cfaae1688b3b8e2162_SeqId_4_ 2023-08-10T06:28:41,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x16b7dc99 to 127.0.0.1:61704 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:28:41,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@896b068, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:28:41,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:28:41,914 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:48462, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:28:41,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:28:41,920 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:38868, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:28:41,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] impl.BackupSystemTable(408): write bulk load descriptor to backup test-1691648868734 with 1 entries 2023-08-10T06:28:41,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] impl.BackupSystemTable(1582): writing raw bulk path hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/acd4d10145cd83e849c8f35f15dd289e/f/eb3ec537e8f043cfaae1688b3b8e2162_SeqId_4_ for test-1691648868734 acd4d10145cd83e849c8f35f15dd289e 2023-08-10T06:28:41,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] impl.BackupSystemTable(415): written 1 rows for bulk load of test-1691648868734 2023-08-10T06:28:41,938 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] client.ConnectionImplementation(2114): Closing master protocol: MasterService 2023-08-10T06:28:41,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x16b7dc99 to 127.0.0.1:61704 2023-08-10T06:28:41,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:28:41,940 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/staging/jenkins__test-1691648868734__7thi6i6edovl5uuqodr6v7roa5899ajf1rc4p1h6cm4t6og2pes8c25gsufbnp22/f/4331be33de5e43ab8d334bb624764b87 into acd4d10145cd83e849c8f35f15dd289e/f as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/acd4d10145cd83e849c8f35f15dd289e/f/eb3ec537e8f043cfaae1688b3b8e2162_SeqId_4_ - updating store file list. 2023-08-10T06:28:41,946 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/acd4d10145cd83e849c8f35f15dd289e/f/eb3ec537e8f043cfaae1688b3b8e2162_SeqId_4_ into acd4d10145cd83e849c8f35f15dd289e/f 2023-08-10T06:28:41,946 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/staging/jenkins__test-1691648868734__7thi6i6edovl5uuqodr6v7roa5899ajf1rc4p1h6cm4t6og2pes8c25gsufbnp22/f/4331be33de5e43ab8d334bb624764b87 into acd4d10145cd83e849c8f35f15dd289e/f (new location: hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/acd4d10145cd83e849c8f35f15dd289e/f/eb3ec537e8f043cfaae1688b3b8e2162_SeqId_4_) 2023-08-10T06:28:41,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/staging/jenkins__test-1691648868734__7thi6i6edovl5uuqodr6v7roa5899ajf1rc4p1h6cm4t6og2pes8c25gsufbnp22/f/4331be33de5e43ab8d334bb624764b87 2023-08-10T06:28:41,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x41ab124e to 127.0.0.1:61704 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:28:41,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7db7a57b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:28:41,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:28:41,955 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:48464, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-08-10T06:28:41,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:28:41,960 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:38878, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-08-10T06:28:41,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] impl.BackupSystemTable(388): write bulk load descriptor to backup test-1691648868734 with 1 entries 2023-08-10T06:28:41,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] impl.BackupSystemTable(1503): writing done bulk path hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/acd4d10145cd83e849c8f35f15dd289e/f/eb3ec537e8f043cfaae1688b3b8e2162_SeqId_4_ for test-1691648868734 acd4d10145cd83e849c8f35f15dd289e 2023-08-10T06:28:41,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] impl.BackupSystemTable(394): written 1 rows for bulk load of test-1691648868734 2023-08-10T06:28:41,994 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] client.ConnectionImplementation(2114): Closing master protocol: MasterService 2023-08-10T06:28:41,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x41ab124e to 127.0.0.1:61704 2023-08-10T06:28:41,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:28:41,998 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] client.ConnectionImplementation(2114): Closing master protocol: MasterService 2023-08-10T06:28:41,999 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0af77597 to 127.0.0.1:61704 2023-08-10T06:28:41,999 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:28:41,999 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-08-10T06:28:41,999 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-08-10T06:28:41,999 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(233): Backup: backup_1691648907578 hdfs://localhost:45379/backupUT/backup_1691648907578/default/test-1691648868734/ 2023-08-10T06:28:41,999 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-08-10T06:28:42,021 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreSingleOverwrite Thread=597 (was 597), OpenFileDescriptor=687 (was 675) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=222 (was 212) - SystemLoadAverage LEAK? -, ProcessCount=171 (was 171), AvailableMemoryMB=2010 (was 2469) 2023-08-10T06:28:42,021 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(130): Thread=597 is superior to 500 2023-08-10T06:28:42,036 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreSingleDNECommand Thread=597, OpenFileDescriptor=687, MaxFileDescriptor=60000, SystemLoadAverage=222, ProcessCount=171, AvailableMemoryMB=2010 2023-08-10T06:28:42,037 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(130): Thread=597 is superior to 500 2023-08-10T06:28:42,037 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] backup.TestFullRestore(280): test restore fails on a single table that does not exist: command-line 2023-08-10T06:28:42,037 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x6f3019e1 to 127.0.0.1:61704 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:28:42,042 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20bbb461, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:28:42,046 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:28:42,048 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:38886, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:28:42,049 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:28:42,050 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:48476, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:28:42,063 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-08-10T06:28:42,069 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:28:42,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691648922069 (current time:1691648922069). 2023-08-10T06:28:42,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:28:42,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-08-10T06:28:42,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x37f3b7fb to 127.0.0.1:61704 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:28:42,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28429440, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:28:42,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:28:42,077 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:38890, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:28:42,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x37f3b7fb to 127.0.0.1:61704 2023-08-10T06:28:42,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:28:42,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:28:42,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=60, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:28:42,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 60 2023-08-10T06:28:42,081 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:28:42,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=60 2023-08-10T06:28:42,082 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:28:42,085 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:28:42,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=60 2023-08-10T06:28:42,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=60 2023-08-10T06:28:42,491 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:28:42,491 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72}] 2023-08-10T06:28:42,495 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=61, ppid=60, state=RUNNABLE; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:28:42,498 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-08-10T06:28:42,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45101 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=61 2023-08-10T06:28:42,646 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. 2023-08-10T06:28:42,646 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(2819): Flushing b9c493f2c9fbef45de19e8268a534e72 2/2 column families, dataSize=1.14 KB heapSize=2.22 KB 2023-08-10T06:28:42,651 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/meta/9d9a4feb1c4e44d99cc63e3f68878731 is 169, key is trslm:hdfs://localhost:45379/backupUT\x00test-1691648868734/meta:log-roll-map/1691648916344/Put/seqid=0 2023-08-10T06:28:42,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=60 2023-08-10T06:28:43,056 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=451 B at sequenceid=28 (bloomFilter=true), to=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/meta/9d9a4feb1c4e44d99cc63e3f68878731 2023-08-10T06:28:43,063 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 9d9a4feb1c4e44d99cc63e3f68878731 2023-08-10T06:28:43,066 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/session/883911c19cab473f8bf801f934c90bc1 is 452, key is session:backup_1691648907578/session:context/1691648916778/Put/seqid=0 2023-08-10T06:28:43,123 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 7ccd561208a3736e53c95b12322d6daf changed from -1.0 to 0.0, refreshing cache 2023-08-10T06:28:43,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=60 2023-08-10T06:28:43,471 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=721 B at sequenceid=28 (bloomFilter=true), to=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/session/883911c19cab473f8bf801f934c90bc1 2023-08-10T06:28:43,477 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 883911c19cab473f8bf801f934c90bc1 2023-08-10T06:28:43,478 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/meta/9d9a4feb1c4e44d99cc63e3f68878731 as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/9d9a4feb1c4e44d99cc63e3f68878731 2023-08-10T06:28:43,483 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/9d9a4feb1c4e44d99cc63e3f68878731, entries=4, sequenceid=28, filesize=5.5 K 2023-08-10T06:28:43,484 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/session/883911c19cab473f8bf801f934c90bc1 as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/883911c19cab473f8bf801f934c90bc1 2023-08-10T06:28:43,490 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/883911c19cab473f8bf801f934c90bc1, entries=2, sequenceid=28, filesize=5.4 K 2023-08-10T06:28:43,491 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1172, heapSize ~2.19 KB/2240, currentSize=0 B/0 for b9c493f2c9fbef45de19e8268a534e72 in 845ms, sequenceid=28, compaction requested=true 2023-08-10T06:28:43,491 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(2520): Flush status journal for b9c493f2c9fbef45de19e8268a534e72: 2023-08-10T06:28:43,491 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. for snapshot_backup_system completed. 2023-08-10T06:28:43,491 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72.' region-info for snapshot=snapshot_backup_system 2023-08-10T06:28:43,491 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:28:43,491 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/bc10682e7f36491d941e2fe62e2e4100, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/9d9a4feb1c4e44d99cc63e3f68878731] hfiles 2023-08-10T06:28:43,491 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/bc10682e7f36491d941e2fe62e2e4100 for snapshot=snapshot_backup_system 2023-08-10T06:28:43,491 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/9d9a4feb1c4e44d99cc63e3f68878731 for snapshot=snapshot_backup_system 2023-08-10T06:28:43,491 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/e88678d5b4b14eecabac817daf12539e, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/37561b9a8ae2406bb6d053c33aeff31c, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/883911c19cab473f8bf801f934c90bc1] hfiles 2023-08-10T06:28:43,491 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/e88678d5b4b14eecabac817daf12539e for snapshot=snapshot_backup_system 2023-08-10T06:28:43,491 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/37561b9a8ae2406bb6d053c33aeff31c for snapshot=snapshot_backup_system 2023-08-10T06:28:43,491 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/883911c19cab473f8bf801f934c90bc1 for snapshot=snapshot_backup_system 2023-08-10T06:28:43,897 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. 2023-08-10T06:28:43,897 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=61 2023-08-10T06:28:43,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster(3994): Remote procedure done, pid=61 2023-08-10T06:28:43,897 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:28:43,899 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=61, ppid=60, state=RUNNABLE; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:28:43,902 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=61, resume processing ppid=60 2023-08-10T06:28:43,902 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=61, ppid=60, state=SUCCESS; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72 in 1.4090 sec 2023-08-10T06:28:43,902 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:28:43,902 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:28:43,903 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:28:43,903 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-08-10T06:28:43,904 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_backup_system 2023-08-10T06:28:44,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=60 2023-08-10T06:28:44,314 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:28:44,322 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:28:44,323 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_backup_system 2023-08-10T06:28:45,143 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:28:45,143 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 60 2023-08-10T06:28:45,144 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=60, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 3.0640 sec 2023-08-10T06:28:46,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=60 2023-08-10T06:28:46,189 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: backup:system, procId: 60 completed 2023-08-10T06:28:46,189 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(115): Backup backup_1691648922042 started at 1691648926189. 2023-08-10T06:28:46,191 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(119): Backup session backup_1691648922042 has been started. 2023-08-10T06:28:46,195 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-08-10T06:28:46,196 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-08-10T06:28:46,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-08-10T06:28:46,196 INFO [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-08-10T06:28:46,196 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-08-10T06:28:46,197 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-08-10T06:28:46,197 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-08-10T06:28:46,198 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-08-10T06:28:46,198 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:28:46,200 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45101-0x10060ea6e8b0001, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-08-10T06:28:46,200 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:46,200 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:28:46,200 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:28:46,201 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:46,201 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-08-10T06:28:46,201 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-08-10T06:28:46,201 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45101-0x10060ea6e8b0001, quorum=127.0.0.1:61704, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-08-10T06:28:46,201 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-08-10T06:28:46,201 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:28:46,201 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-08-10T06:28:46,201 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-08-10T06:28:46,202 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-08-10T06:28:46,202 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-08-10T06:28:46,202 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-08-10T06:28:46,203 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-08-10T06:28:46,203 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-08-10T06:28:46,203 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,45101,1691648859755' joining acquired barrier for procedure (rolllog) in zk 2023-08-10T06:28:46,205 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:46,205 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-08-10T06:28:46,205 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:46,205 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:28:46,205 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:28:46,205 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45101-0x10060ea6e8b0001, quorum=127.0.0.1:61704, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-08-10T06:28:46,205 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-08-10T06:28:46,205 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:28:46,206 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:28:46,206 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:28:46,206 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:46,206 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:28:46,207 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,45101,1691648859755' joining acquired barrier for procedure 'rolllog' on coordinator 2023-08-10T06:28:46,207 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@3885d29b[Count = 0] remaining members to acquire global barrier 2023-08-10T06:28:46,207 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-08-10T06:28:46,207 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-08-10T06:28:46,208 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45101-0x10060ea6e8b0001, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-08-10T06:28:46,208 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-08-10T06:28:46,208 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-08-10T06:28:46,208 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-08-10T06:28:46,208 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:46,208 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-08-10T06:28:46,208 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-08-10T06:28:46,208 DEBUG [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:46,208 INFO [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1691648911777 highest: 1691648911777 on jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:46,209 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta:.meta(num 1691648911758) roll requested 2023-08-10T06:28:46,222 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691648926209.meta, exclude list is [], retry=0 2023-08-10T06:28:46,226 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK] 2023-08-10T06:28:46,228 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691648911758.meta with entries=12, filesize=3.07 KB; new WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691648926209.meta 2023-08-10T06:28:46,228 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK]] 2023-08-10T06:28:46,229 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691648911758.meta is not closed yet, will try archiving it next time 2023-08-10T06:28:46,229 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C45101%2C1691648859755:(num 1691648911777) roll requested 2023-08-10T06:28:46,231 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691648911758.meta not finished, retry = 0 2023-08-10T06:28:46,240 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691648926229, exclude list is [], retry=0 2023-08-10T06:28:46,244 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK] 2023-08-10T06:28:46,246 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691648911777 with entries=17, filesize=5.25 KB; new WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691648926229 2023-08-10T06:28:46,246 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK]] 2023-08-10T06:28:46,246 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691648911777 is not closed yet, will try archiving it next time 2023-08-10T06:28:46,249 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691648911777 not finished, retry = 0 2023-08-10T06:28:46,249 DEBUG [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-08-10T06:28:46,249 INFO [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1691648926229 on jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:46,259 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-08-10T06:28:46,259 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,45101,1691648859755' in zk 2023-08-10T06:28:46,263 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-08-10T06:28:46,263 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:46,263 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:28:46,263 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-08-10T06:28:46,264 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:46,264 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:28:46,264 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:28:46,264 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:28:46,265 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:28:46,265 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:28:46,265 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:46,266 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:28:46,266 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:28:46,266 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:46,267 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,45101,1691648859755': 2023-08-10T06:28:46,267 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,45101,1691648859755' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-08-10T06:28:46,267 INFO [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-08-10T06:28:46,267 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-08-10T06:28:46,267 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-08-10T06:28:46,267 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-08-10T06:28:46,267 INFO [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-08-10T06:28:46,268 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-08-10T06:28:46,268 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45101-0x10060ea6e8b0001, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-08-10T06:28:46,269 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45101-0x10060ea6e8b0001, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-08-10T06:28:46,269 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-08-10T06:28:46,269 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-08-10T06:28:46,269 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:28:46,269 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:28:46,269 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:28:46,269 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:28:46,269 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:28:46,269 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:28:46,269 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:46,269 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:28:46,269 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:28:46,270 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:28:46,270 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:28:46,271 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:46,271 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:46,271 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:28:46,271 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:28:46,272 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:46,275 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45101-0x10060ea6e8b0001, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-08-10T06:28:46,275 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45101-0x10060ea6e8b0001, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-08-10T06:28:46,275 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:28:46,275 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:46,275 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:28:46,275 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:28:46,275 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-08-10T06:28:46,275 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-08-10T06:28:46,275 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-08-10T06:28:46,276 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-08-10T06:28:46,276 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:46,276 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-08-10T06:28:46,276 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-08-10T06:28:46,276 DEBUG [Listener at localhost/36815-EventThread {}] zookeeper.ZKWatcher(604): master:33705-0x10060ea6e8b0000, quorum=127.0.0.1:61704, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-08-10T06:28:46,276 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:28:46,276 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:28:46,276 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] client.HBaseAdmin(2751): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-08-10T06:28:46,276 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] client.HBaseAdmin(2759): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-08-10T06:28:46,377 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] client.HBaseAdmin(2765): Getting current status of procedure from master... 2023-08-10T06:28:46,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-08-10T06:28:46,380 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1691648926380_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } 2023-08-10T06:28:46,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691648926380 (current time:1691648926380). 2023-08-10T06:28:46,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:28:46,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1691648926380_default_test-1691648868734 VERSION not specified, setting to 2 2023-08-10T06:28:46,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x56fee9a5 to 127.0.0.1:61704 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:28:46,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c18d0ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:28:46,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:28:46,389 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:38892, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:28:46,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x56fee9a5 to 127.0.0.1:61704 2023-08-10T06:28:46,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:28:46,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:28:46,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=62, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1691648926380_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } 2023-08-10T06:28:46,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1691648926380_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 }, snapshot procedure id = 62 2023-08-10T06:28:46,392 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1691648926380_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:28:46,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=62 2023-08-10T06:28:46,393 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1691648926380_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:28:46,395 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1691648926380_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:28:46,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=62 2023-08-10T06:28:46,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=62 2023-08-10T06:28:46,802 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1691648926380_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:28:46,802 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE; SnapshotRegionProcedure acd4d10145cd83e849c8f35f15dd289e}] 2023-08-10T06:28:46,805 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=63, ppid=62, state=RUNNABLE; SnapshotRegionProcedure acd4d10145cd83e849c8f35f15dd289e 2023-08-10T06:28:46,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45101 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=63 2023-08-10T06:28:46,956 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1691648868734,,1691648917151.acd4d10145cd83e849c8f35f15dd289e. 2023-08-10T06:28:46,956 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.HRegion(2520): Flush status journal for acd4d10145cd83e849c8f35f15dd289e: 2023-08-10T06:28:46,956 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1691648868734,,1691648917151.acd4d10145cd83e849c8f35f15dd289e. for snapshot_1691648926380_default_test-1691648868734 completed. 2023-08-10T06:28:46,957 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(240): Storing 'test-1691648868734,,1691648917151.acd4d10145cd83e849c8f35f15dd289e.' region-info for snapshot=snapshot_1691648926380_default_test-1691648868734 2023-08-10T06:28:46,957 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:28:46,957 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/acd4d10145cd83e849c8f35f15dd289e/f/eb3ec537e8f043cfaae1688b3b8e2162_SeqId_4_] hfiles 2023-08-10T06:28:46,957 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/acd4d10145cd83e849c8f35f15dd289e/f/eb3ec537e8f043cfaae1688b3b8e2162_SeqId_4_ for snapshot=snapshot_1691648926380_default_test-1691648868734 2023-08-10T06:28:46,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=62 2023-08-10T06:28:47,363 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1691648868734,,1691648917151.acd4d10145cd83e849c8f35f15dd289e. 2023-08-10T06:28:47,363 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=63 2023-08-10T06:28:47,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster(3994): Remote procedure done, pid=63 2023-08-10T06:28:47,364 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1691648926380_default_test-1691648868734 on region acd4d10145cd83e849c8f35f15dd289e 2023-08-10T06:28:47,365 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=63, ppid=62, state=RUNNABLE; SnapshotRegionProcedure acd4d10145cd83e849c8f35f15dd289e 2023-08-10T06:28:47,367 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=63, resume processing ppid=62 2023-08-10T06:28:47,367 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=63, ppid=62, state=SUCCESS; SnapshotRegionProcedure acd4d10145cd83e849c8f35f15dd289e in 564 msec 2023-08-10T06:28:47,367 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1691648926380_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:28:47,368 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1691648926380_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:28:47,369 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1691648926380_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:28:47,369 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1691648926380_default_test-1691648868734 2023-08-10T06:28:47,369 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_1691648926380_default_test-1691648868734 2023-08-10T06:28:47,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=62 2023-08-10T06:28:47,780 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1691648926380_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:28:47,786 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1691648926380_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:28:47,786 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_1691648926380_default_test-1691648868734 to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691648926380_default_test-1691648868734 2023-08-10T06:28:48,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=62 2023-08-10T06:28:48,604 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1691648926380_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:28:48,604 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1691648926380_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 }, snapshot procedure id = 62 2023-08-10T06:28:48,605 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=62, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1691648926380_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } in 2.2140 sec 2023-08-10T06:28:50,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=62 2023-08-10T06:28:50,500 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: default:test-1691648868734, procId: 62 completed 2023-08-10T06:28:50,500 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1691648922042 2023-08-10T06:28:50,500 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-08-10T06:28:50,500 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-08-10T06:28:50,500 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1691648922042_test-1691648868734 2023-08-10T06:28:50,501 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1691648926380_default_test-1691648868734 to hdfs://localhost:45379/backupUT/backup_1691648922042/default/test-1691648868734/ 2023-08-10T06:28:50,501 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-08-10T06:28:50,524 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:45379, inputRoot=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e 2023-08-10T06:28:50,524 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,33705,1691648856595_-425815215_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:45379/backupUT/backup_1691648922042/default/test-1691648868734, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:45379/backupUT/backup_1691648922042/default/test-1691648868734/.hbase-snapshot/.tmp/snapshot_1691648926380_default_test-1691648868734 2023-08-10T06:28:50,524 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:45379, inputRoot=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e, snapshotDir=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691648926380_default_test-1691648868734. 2023-08-10T06:28:50,536 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691648926380_default_test-1691648868734 to hdfs://localhost:45379/backupUT/backup_1691648922042/default/test-1691648868734/.hbase-snapshot/.tmp/snapshot_1691648926380_default_test-1691648868734 2023-08-10T06:28:50,952 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:50,952 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:50,953 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:50,953 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:51,928 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/hadoop-5994877035816461339.jar 2023-08-10T06:28:51,929 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:51,929 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:51,930 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:51,930 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:51,931 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:51,931 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:51,931 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:51,932 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:51,932 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-08-10T06:28:51,933 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:28:51,934 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-08-10T06:28:51,935 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-08-10T06:28:51,936 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-08-10T06:28:51,937 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-08-10T06:28:51,937 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-08-10T06:28:51,938 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-08-10T06:28:51,938 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-08-10T06:28:51,939 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-08-10T06:28:51,940 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-08-10T06:28:51,940 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-08-10T06:28:51,941 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-08-10T06:28:51,942 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:28:51,943 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:28:51,944 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:28:51,944 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:28:51,944 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:28:51,945 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:28:51,946 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:28:51,975 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-08-10T06:28:51,993 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1691648926380_default_test-1691648868734' hfile list 2023-08-10T06:28:51,995 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-08-10T06:28:52,062 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691648932030/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-08-10T06:28:52,062 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691648932030/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-08-10T06:28:52,114 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-08-10T06:28:52,139 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/acd4d10145cd83e849c8f35f15dd289e/f/eb3ec537e8f043cfaae1688b3b8e2162_SeqId_4_ output=hdfs://localhost:45379/backupUT/backup_1691648922042/default/test-1691648868734/archive/data/default/test-1691648868734/acd4d10145cd83e849c8f35f15dd289e/f/eb3ec537e8f043cfaae1688b3b8e2162_SeqId_4_ 2023-08-10T06:28:52,139 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8724 (8.5 K) time=0sec 4.160M/sec 2023-08-10T06:28:53,097 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-08-10T06:28:53,098 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-08-10T06:28:53,105 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1691648926380_default_test-1691648868734 2023-08-10T06:28:53,105 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1691648926380_default_test-1691648868734 finished. 2023-08-10T06:28:53,105 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(1014): test-1691648868734 2023-08-10T06:28:53,111 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691648922042 2023-08-10T06:28:53,111 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-08-10T06:28:53,517 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:45379/backupUT/backup_1691648922042/.backup.manifest 2023-08-10T06:28:53,517 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(358): Backup backup_1691648922042 finished: type=FULL,tablelist=test-1691648868734,targetRootDir=hdfs://localhost:45379/backupUT,startts=1691648926189,completets=1691648933111,bytescopied=0 2023-08-10T06:28:53,517 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-08-10T06:28:53,517 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1691648926380_default_test-1691648868734 2023-08-10T06:28:53,518 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1691648926380_default_test-1691648868734" 2023-08-10T06:28:53,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1691648926380_default_test-1691648868734 2023-08-10T06:28:53,521 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1691648926380_default_test-1691648868734 for backup backup_1691648922042 succeeded. 2023-08-10T06:28:53,522 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-08-10T06:28:53,526 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-08-10T06:28:53,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-08-10T06:28:53,528 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-08-10T06:28:53,530 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-08-10T06:28:53,536 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(376): Backup backup_1691648922042 completed. 2023-08-10T06:28:53,536 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] client.ConnectionImplementation(2114): Closing master protocol: MasterService 2023-08-10T06:28:53,536 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x6f3019e1 to 127.0.0.1:61704 2023-08-10T06:28:53,536 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:28:53,547 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] backup.TestFullRestore(285): backup complete You have to specify either set name or table list to restore Usage: hbase restore [options] backup_path Path to a backup destination root backup_id Backup image ID to restore table(s) Comma-separated list of tables to restore Options: -c Check restore sequence and dependencies only (does not execute the command) -d Enable debug loggings -h,--help Show usage -m A comma separated list of target tables. If specified, each table in must have a mapping -o Overwrite data if any of the restore target tables exists -q Yarn queue name to run backup restore command on -s Backup set to restore, mutually exclusive with -t (table list) -t Table name list, comma-separated. To enable backup, in hbase-site.xml, set: hbase.backup.enable=true hbase.master.logcleaner.plugins=YOUR_PLUGINS,org.apache.hadoop.hbase.backup.master.BackupLogCleaner hbase.procedure.master.classes=YOUR_CLASSES,org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager hbase.procedure.regionserver.classes=YOUR_CLASSES,org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager hbase.coprocessor.region.classes=YOUR_CLASSES,org.apache.hadoop.hbase.backup.BackupObserver and restart the cluster For more information please see http://hbase.apache.org/book.html#backuprestore 2023-08-10T06:28:53,577 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreSingleDNECommand Thread=591 (was 597), OpenFileDescriptor=674 (was 687), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=188 (was 222), ProcessCount=171 (was 171), AvailableMemoryMB=1988 (was 2010) 2023-08-10T06:28:53,577 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(130): Thread=591 is superior to 500 2023-08-10T06:28:53,595 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreCheckCommand Thread=591, OpenFileDescriptor=674, MaxFileDescriptor=60000, SystemLoadAverage=188, ProcessCount=171, AvailableMemoryMB=1987 2023-08-10T06:28:53,596 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(130): Thread=591 is superior to 500 2023-08-10T06:28:53,596 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] backup.TestFullRestore(97): test full restore on a single table: command-line, check only 2023-08-10T06:28:53,604 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7073c3fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:28:53,608 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:28:53,610 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:57636, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:28:53,612 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:28:53,613 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:52498, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:28:53,624 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-08-10T06:28:53,629 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:28:53,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691648933629 (current time:1691648933629). 2023-08-10T06:28:53,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:28:53,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-08-10T06:28:53,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ad25a87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:28:53,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:28:53,636 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:57652, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:28:53,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:28:53,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:28:53,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=64, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:28:53,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 64 2023-08-10T06:28:53,640 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:28:53,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=64 2023-08-10T06:28:53,641 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:28:53,642 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:28:53,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=64 2023-08-10T06:28:53,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=64 2023-08-10T06:28:54,049 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:28:54,049 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72}] 2023-08-10T06:28:54,051 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:28:54,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45101 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=65 2023-08-10T06:28:54,202 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. 2023-08-10T06:28:54,203 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(2819): Flushing b9c493f2c9fbef45de19e8268a534e72 2/2 column families, dataSize=1.14 KB heapSize=2.22 KB 2023-08-10T06:28:54,206 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/meta/f04a99990b2c429eaf1096ef413be3b8 is 169, key is trslm:hdfs://localhost:45379/backupUT\x00test-1691648868734/meta:log-roll-map/1691648933107/Put/seqid=0 2023-08-10T06:28:54,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=64 2023-08-10T06:28:54,611 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=451 B at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/meta/f04a99990b2c429eaf1096ef413be3b8 2023-08-10T06:28:54,617 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: f04a99990b2c429eaf1096ef413be3b8 2023-08-10T06:28:54,620 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/session/0c7705a45c554d6a97fa37fac319ceff is 452, key is session:backup_1691648922042/session:context/1691648933529/Put/seqid=0 2023-08-10T06:28:54,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=64 2023-08-10T06:28:55,025 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=721 B at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/session/0c7705a45c554d6a97fa37fac319ceff 2023-08-10T06:28:55,030 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 0c7705a45c554d6a97fa37fac319ceff 2023-08-10T06:28:55,031 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/meta/f04a99990b2c429eaf1096ef413be3b8 as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/f04a99990b2c429eaf1096ef413be3b8 2023-08-10T06:28:55,036 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/f04a99990b2c429eaf1096ef413be3b8, entries=4, sequenceid=39, filesize=5.5 K 2023-08-10T06:28:55,037 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/session/0c7705a45c554d6a97fa37fac319ceff as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/0c7705a45c554d6a97fa37fac319ceff 2023-08-10T06:28:55,042 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/0c7705a45c554d6a97fa37fac319ceff, entries=2, sequenceid=39, filesize=5.4 K 2023-08-10T06:28:55,043 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1172, heapSize ~2.19 KB/2240, currentSize=0 B/0 for b9c493f2c9fbef45de19e8268a534e72 in 840ms, sequenceid=39, compaction requested=true 2023-08-10T06:28:55,043 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(2520): Flush status journal for b9c493f2c9fbef45de19e8268a534e72: 2023-08-10T06:28:55,043 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. for snapshot_backup_system completed. 2023-08-10T06:28:55,043 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72.' region-info for snapshot=snapshot_backup_system 2023-08-10T06:28:55,043 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:28:55,043 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/bc10682e7f36491d941e2fe62e2e4100, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/9d9a4feb1c4e44d99cc63e3f68878731, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/f04a99990b2c429eaf1096ef413be3b8] hfiles 2023-08-10T06:28:55,043 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/bc10682e7f36491d941e2fe62e2e4100 for snapshot=snapshot_backup_system 2023-08-10T06:28:55,043 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/9d9a4feb1c4e44d99cc63e3f68878731 for snapshot=snapshot_backup_system 2023-08-10T06:28:55,043 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/f04a99990b2c429eaf1096ef413be3b8 for snapshot=snapshot_backup_system 2023-08-10T06:28:55,043 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/e88678d5b4b14eecabac817daf12539e, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/37561b9a8ae2406bb6d053c33aeff31c, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/883911c19cab473f8bf801f934c90bc1, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/0c7705a45c554d6a97fa37fac319ceff] hfiles 2023-08-10T06:28:55,043 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (1/4): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/e88678d5b4b14eecabac817daf12539e for snapshot=snapshot_backup_system 2023-08-10T06:28:55,043 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (2/4): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/37561b9a8ae2406bb6d053c33aeff31c for snapshot=snapshot_backup_system 2023-08-10T06:28:55,043 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (3/4): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/883911c19cab473f8bf801f934c90bc1 for snapshot=snapshot_backup_system 2023-08-10T06:28:55,043 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (4/4): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/0c7705a45c554d6a97fa37fac319ceff for snapshot=snapshot_backup_system 2023-08-10T06:28:55,448 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. 2023-08-10T06:28:55,448 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=65 2023-08-10T06:28:55,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster(3994): Remote procedure done, pid=65 2023-08-10T06:28:55,449 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:28:55,450 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:28:55,452 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=65, resume processing ppid=64 2023-08-10T06:28:55,452 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:28:55,452 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=65, ppid=64, state=SUCCESS; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72 in 1.4020 sec 2023-08-10T06:28:55,452 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:28:55,453 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:28:55,453 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-08-10T06:28:55,454 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_backup_system 2023-08-10T06:28:55,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=64 2023-08-10T06:28:55,863 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:28:55,872 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:28:55,872 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_backup_system 2023-08-10T06:28:56,690 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:28:56,690 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 64 2023-08-10T06:28:56,692 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=64, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 3.0520 sec 2023-08-10T06:28:57,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=64 2023-08-10T06:28:57,746 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(115): Backup backup_1691648933604 started at 1691648937746. 2023-08-10T06:28:57,748 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(119): Backup session backup_1691648933604 has been started. 2023-08-10T06:28:57,753 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-08-10T06:28:57,753 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-08-10T06:28:57,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure.ProcedureCoordinator(143): Procedure rolllog was in running list but was completed. Accepting new attempt. 2023-08-10T06:28:57,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-08-10T06:28:57,754 INFO [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-08-10T06:28:57,754 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-08-10T06:28:57,755 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-08-10T06:28:57,755 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-08-10T06:28:57,755 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:28:57,757 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:57,757 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:28:57,757 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:28:57,758 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-08-10T06:28:57,758 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-08-10T06:28:57,758 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-08-10T06:28:57,758 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:28:57,758 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-08-10T06:28:57,759 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-08-10T06:28:57,759 WARN [zk-event-processor-pool-0 {}] procedure.ProcedureMember(133): A completed old subproc rolllog is still present, removing 2023-08-10T06:28:57,759 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-08-10T06:28:57,759 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-08-10T06:28:57,759 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-08-10T06:28:57,760 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-08-10T06:28:57,760 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-08-10T06:28:57,760 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,45101,1691648859755' joining acquired barrier for procedure (rolllog) in zk 2023-08-10T06:28:57,762 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-08-10T06:28:57,762 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:57,762 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:28:57,762 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:28:57,762 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-08-10T06:28:57,762 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:28:57,763 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:28:57,763 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:28:57,763 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:57,763 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:28:57,764 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,45101,1691648859755' joining acquired barrier for procedure 'rolllog' on coordinator 2023-08-10T06:28:57,764 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@655fa26d[Count = 0] remaining members to acquire global barrier 2023-08-10T06:28:57,764 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-08-10T06:28:57,764 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-08-10T06:28:57,765 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-08-10T06:28:57,765 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-08-10T06:28:57,765 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-08-10T06:28:57,765 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-08-10T06:28:57,765 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-08-10T06:28:57,765 DEBUG [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:57,765 INFO [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1691648926229 highest: 1691648926229 on jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:57,765 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta:.meta(num 1691648926209) roll requested 2023-08-10T06:28:57,777 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691648937765.meta, exclude list is [], retry=0 2023-08-10T06:28:57,781 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK] 2023-08-10T06:28:57,783 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691648926209.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691648937765.meta 2023-08-10T06:28:57,783 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK]] 2023-08-10T06:28:57,783 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691648926209.meta is not closed yet, will try archiving it next time 2023-08-10T06:28:57,783 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C45101%2C1691648859755:(num 1691648926229) roll requested 2023-08-10T06:28:57,785 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691648926209.meta not finished, retry = 0 2023-08-10T06:28:57,794 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691648937783, exclude list is [], retry=0 2023-08-10T06:28:57,797 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK] 2023-08-10T06:28:57,798 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691648926229 with entries=10, filesize=2.31 KB; new WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691648937783 2023-08-10T06:28:57,799 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK]] 2023-08-10T06:28:57,799 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691648926229 is not closed yet, will try archiving it next time 2023-08-10T06:28:57,806 DEBUG [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-08-10T06:28:57,806 INFO [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1691648937783 on jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:57,815 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-08-10T06:28:57,815 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,45101,1691648859755' in zk 2023-08-10T06:28:57,817 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-08-10T06:28:57,817 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:28:57,817 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-08-10T06:28:57,817 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:57,818 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:28:57,818 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:28:57,819 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:28:57,819 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:28:57,819 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:28:57,820 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:57,820 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:28:57,821 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:28:57,821 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:57,822 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,45101,1691648859755': 2023-08-10T06:28:57,822 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,45101,1691648859755' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-08-10T06:28:57,822 INFO [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-08-10T06:28:57,822 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-08-10T06:28:57,822 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-08-10T06:28:57,822 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-08-10T06:28:57,822 INFO [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-08-10T06:28:57,825 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-08-10T06:28:57,825 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-08-10T06:28:57,825 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:28:57,825 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:28:57,825 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:28:57,826 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:28:57,826 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:28:57,826 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:28:57,826 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:28:57,826 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:28:57,827 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:28:57,827 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:28:57,828 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:57,828 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:28:57,829 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:28:57,829 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:28:57,833 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:28:57,833 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:28:57,833 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:28:57,833 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-08-10T06:28:57,833 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-08-10T06:28:57,834 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:28:57,834 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:28:57,886 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691648926209.meta to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/oldWALs/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691648926209.meta 2023-08-10T06:28:57,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-08-10T06:28:57,938 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1691648937937_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } 2023-08-10T06:28:57,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691648937938 (current time:1691648937938). 2023-08-10T06:28:57,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:28:57,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1691648937937_default_test-1691648868734 VERSION not specified, setting to 2 2023-08-10T06:28:57,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ea910e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:28:57,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:28:57,946 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:57668, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:28:57,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:28:57,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:28:57,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=66, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1691648937937_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } 2023-08-10T06:28:57,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1691648937937_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 }, snapshot procedure id = 66 2023-08-10T06:28:57,950 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1691648937937_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:28:57,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=66 2023-08-10T06:28:57,951 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1691648937937_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:28:57,953 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1691648937937_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:28:58,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=66 2023-08-10T06:28:58,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=66 2023-08-10T06:28:58,358 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1691648937937_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:28:58,359 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE; SnapshotRegionProcedure acd4d10145cd83e849c8f35f15dd289e}] 2023-08-10T06:28:58,360 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=67, ppid=66, state=RUNNABLE; SnapshotRegionProcedure acd4d10145cd83e849c8f35f15dd289e 2023-08-10T06:28:58,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45101 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=67 2023-08-10T06:28:58,512 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1691648868734,,1691648917151.acd4d10145cd83e849c8f35f15dd289e. 2023-08-10T06:28:58,512 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.HRegion(2520): Flush status journal for acd4d10145cd83e849c8f35f15dd289e: 2023-08-10T06:28:58,512 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1691648868734,,1691648917151.acd4d10145cd83e849c8f35f15dd289e. for snapshot_1691648937937_default_test-1691648868734 completed. 2023-08-10T06:28:58,512 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] snapshot.SnapshotManifest(240): Storing 'test-1691648868734,,1691648917151.acd4d10145cd83e849c8f35f15dd289e.' region-info for snapshot=snapshot_1691648937937_default_test-1691648868734 2023-08-10T06:28:58,512 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:28:58,512 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/acd4d10145cd83e849c8f35f15dd289e/f/eb3ec537e8f043cfaae1688b3b8e2162_SeqId_4_] hfiles 2023-08-10T06:28:58,512 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/acd4d10145cd83e849c8f35f15dd289e/f/eb3ec537e8f043cfaae1688b3b8e2162_SeqId_4_ for snapshot=snapshot_1691648937937_default_test-1691648868734 2023-08-10T06:28:58,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=66 2023-08-10T06:28:58,918 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1691648868734,,1691648917151.acd4d10145cd83e849c8f35f15dd289e. 2023-08-10T06:28:58,918 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=67 2023-08-10T06:28:58,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster(3994): Remote procedure done, pid=67 2023-08-10T06:28:58,919 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1691648937937_default_test-1691648868734 on region acd4d10145cd83e849c8f35f15dd289e 2023-08-10T06:28:58,920 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=67, ppid=66, state=RUNNABLE; SnapshotRegionProcedure acd4d10145cd83e849c8f35f15dd289e 2023-08-10T06:28:58,922 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=67, resume processing ppid=66 2023-08-10T06:28:58,922 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=67, ppid=66, state=SUCCESS; SnapshotRegionProcedure acd4d10145cd83e849c8f35f15dd289e in 562 msec 2023-08-10T06:28:58,922 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1691648937937_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:28:58,923 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1691648937937_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:28:58,923 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1691648937937_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:28:58,923 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1691648937937_default_test-1691648868734 2023-08-10T06:28:58,924 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_1691648937937_default_test-1691648868734 2023-08-10T06:28:59,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=66 2023-08-10T06:28:59,333 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1691648937937_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:28:59,339 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1691648937937_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:28:59,339 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_1691648937937_default_test-1691648868734 to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691648937937_default_test-1691648868734 2023-08-10T06:29:00,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=66 2023-08-10T06:29:00,156 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1691648937937_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:29:00,156 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1691648937937_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 }, snapshot procedure id = 66 2023-08-10T06:29:00,157 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=66, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1691648937937_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } in 2.2090 sec 2023-08-10T06:29:02,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=66 2023-08-10T06:29:02,057 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1691648933604 2023-08-10T06:29:02,057 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-08-10T06:29:02,057 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-08-10T06:29:02,057 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1691648933604_test-1691648868734 2023-08-10T06:29:02,057 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1691648937937_default_test-1691648868734 to hdfs://localhost:45379/backupUT/backup_1691648933604/default/test-1691648868734/ 2023-08-10T06:29:02,058 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-08-10T06:29:02,081 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:45379, inputRoot=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e 2023-08-10T06:29:02,081 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,33705,1691648856595_-1406464547_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:45379/backupUT/backup_1691648933604/default/test-1691648868734, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:45379/backupUT/backup_1691648933604/default/test-1691648868734/.hbase-snapshot/.tmp/snapshot_1691648937937_default_test-1691648868734 2023-08-10T06:29:02,081 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:45379, inputRoot=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e, snapshotDir=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691648937937_default_test-1691648868734. 2023-08-10T06:29:02,089 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691648937937_default_test-1691648868734 to hdfs://localhost:45379/backupUT/backup_1691648933604/default/test-1691648868734/.hbase-snapshot/.tmp/snapshot_1691648937937_default_test-1691648868734 2023-08-10T06:29:02,503 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:02,504 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:02,504 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:02,505 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:03,462 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/hadoop-8529332311086420739.jar 2023-08-10T06:29:03,462 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:03,463 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:03,463 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:03,464 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:03,464 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:03,464 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:03,465 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:03,465 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:03,465 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:03,466 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:29:03,467 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-08-10T06:29:03,467 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-08-10T06:29:03,467 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-08-10T06:29:03,468 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-08-10T06:29:03,468 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-08-10T06:29:03,469 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-08-10T06:29:03,469 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-08-10T06:29:03,469 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-08-10T06:29:03,470 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-08-10T06:29:03,470 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-08-10T06:29:03,470 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-08-10T06:29:03,471 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:29:03,471 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:29:03,472 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:29:03,472 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:29:03,473 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:29:03,473 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:29:03,474 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:29:03,516 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1691648937937_default_test-1691648868734' hfile list 2023-08-10T06:29:03,519 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-08-10T06:29:03,587 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691648943554/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-08-10T06:29:03,587 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691648943554/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-08-10T06:29:03,639 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-08-10T06:29:03,687 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/acd4d10145cd83e849c8f35f15dd289e/f/eb3ec537e8f043cfaae1688b3b8e2162_SeqId_4_ output=hdfs://localhost:45379/backupUT/backup_1691648933604/default/test-1691648868734/archive/data/default/test-1691648868734/acd4d10145cd83e849c8f35f15dd289e/f/eb3ec537e8f043cfaae1688b3b8e2162_SeqId_4_ 2023-08-10T06:29:03,687 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8724 (8.5 K) time=0sec 4.160M/sec 2023-08-10T06:29:04,622 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-08-10T06:29:04,623 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-08-10T06:29:04,629 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1691648937937_default_test-1691648868734 2023-08-10T06:29:04,629 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1691648937937_default_test-1691648868734 finished. 2023-08-10T06:29:04,629 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(1014): test-1691648868734 2023-08-10T06:29:04,635 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691648933604 2023-08-10T06:29:04,635 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-08-10T06:29:05,041 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:45379/backupUT/backup_1691648933604/.backup.manifest 2023-08-10T06:29:05,041 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(358): Backup backup_1691648933604 finished: type=FULL,tablelist=test-1691648868734,targetRootDir=hdfs://localhost:45379/backupUT,startts=1691648937746,completets=1691648944635,bytescopied=0 2023-08-10T06:29:05,041 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-08-10T06:29:05,041 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1691648937937_default_test-1691648868734 2023-08-10T06:29:05,042 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1691648937937_default_test-1691648868734" 2023-08-10T06:29:05,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1691648937937_default_test-1691648868734 2023-08-10T06:29:05,045 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1691648937937_default_test-1691648868734 for backup backup_1691648933604 succeeded. 2023-08-10T06:29:05,046 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-08-10T06:29:05,050 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-08-10T06:29:05,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-08-10T06:29:05,053 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-08-10T06:29:05,055 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-08-10T06:29:05,061 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(376): Backup backup_1691648933604 completed. 2023-08-10T06:29:05,061 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:29:05,061 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] backup.TestFullRestore(101): backup complete 2023-08-10T06:29:05,076 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] backup.RestoreDriver(108): Found -check option in restore command, will check and verify the dependencies 2023-08-10T06:29:05,089 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b204848, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:29:05,091 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:45379/backupUT/backup_1691648933604/.backup.manifest 2023-08-10T06:29:05,093 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691648933604/.backup.manifest 2023-08-10T06:29:05,094 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] util.BackupUtils(677): Dependent image(s) from old to new: 2023-08-10T06:29:05,094 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] util.BackupUtils(686): Backup image: backup_1691648933604 for 'test-1691648868734' is available 2023-08-10T06:29:05,094 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupAdminImpl(511): Checking backup images: OK 2023-08-10T06:29:05,094 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:29:05,118 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreCheckCommand Thread=595 (was 591) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1703575583_16 at /127.0.0.1:57944 [Waiting for operation #8] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-937657374_16 at /127.0.0.1:59584 [Receiving block BP-1233959968-172.31.12.81-1691648853138:blk_1073741931_1107] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x65702ead-shared-pool-12 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: WAL-Archive-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-1233959968-172.31.12.81-1691648853138:blk_1073741932_1108, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:61704@0x2b08e230-SendThread(127.0.0.1:61704) java.lang.Thread.sleep(Native Method) org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:228) org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1338) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1261) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:61704@0x2b08e230-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:61704@0x4e8b8e04-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1510243438_16 at /127.0.0.1:59540 [Waiting for operation #22] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:61704@0x2b08e230 java.lang.Object.wait(Native Method) java.lang.Thread.join(Thread.java:1257) java.lang.Thread.join(Thread.java:1331) org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1455) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1485) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1415) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$481/1293446881.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-937657374_16 at /127.0.0.1:59590 [Receiving block BP-1233959968-172.31.12.81-1691648853138:blk_1073741932_1108] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:61704@0x4e8b8e04 java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1529) org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1512) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1481) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1415) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$481/1293446881.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x69012128-shared-pool-30 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x65702ead-shared-pool-13 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:61704@0x4e8b8e04-SendThread(127.0.0.1:61704) sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:345) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1223) Potentially hanging thread: PacketResponder: BP-1233959968-172.31.12.81-1691648853138:blk_1073741931_1107, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=680 (was 674) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=154 (was 188), ProcessCount=171 (was 171), AvailableMemoryMB=1966 (was 1987) 2023-08-10T06:29:05,118 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(130): Thread=595 is superior to 500 2023-08-10T06:29:05,139 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreSingle Thread=595, OpenFileDescriptor=680, MaxFileDescriptor=60000, SystemLoadAverage=154, ProcessCount=171, AvailableMemoryMB=1965 2023-08-10T06:29:05,139 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(130): Thread=595 is superior to 500 2023-08-10T06:29:05,139 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] backup.TestFullRestore(55): test full restore on a single table empty table 2023-08-10T06:29:05,151 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54590c5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:29:05,154 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:29:05,156 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:58646, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:29:05,162 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:29:05,164 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:56600, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:29:05,183 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-08-10T06:29:05,189 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:29:05,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691648945189 (current time:1691648945189). 2023-08-10T06:29:05,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:29:05,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-08-10T06:29:05,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f83c455, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:29:05,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:29:05,198 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:58648, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:29:05,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:29:05,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:29:05,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=68, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:29:05,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 68 2023-08-10T06:29:05,202 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:29:05,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=68 2023-08-10T06:29:05,202 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:29:05,204 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:29:05,210 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:29:05,210 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72}] 2023-08-10T06:29:05,211 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=69, ppid=68, state=RUNNABLE; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:29:05,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=68 2023-08-10T06:29:05,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45101 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=69 2023-08-10T06:29:05,363 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. 2023-08-10T06:29:05,364 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.HRegion(2819): Flushing b9c493f2c9fbef45de19e8268a534e72 2/2 column families, dataSize=1.14 KB heapSize=2.22 KB 2023-08-10T06:29:05,369 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/meta/9e97dc38b66b476f952990b23fa0a3ca is 169, key is trslm:hdfs://localhost:45379/backupUT\x00test-1691648868734/meta:log-roll-map/1691648944631/Put/seqid=0 2023-08-10T06:29:05,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=68 2023-08-10T06:29:05,774 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=451 B at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/meta/9e97dc38b66b476f952990b23fa0a3ca 2023-08-10T06:29:05,780 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 9e97dc38b66b476f952990b23fa0a3ca 2023-08-10T06:29:05,783 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/session/398a15bfffb84acfadb1be252c2e133a is 452, key is session:backup_1691648933604/session:context/1691648945054/Put/seqid=0 2023-08-10T06:29:05,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=68 2023-08-10T06:29:06,188 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=721 B at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/session/398a15bfffb84acfadb1be252c2e133a 2023-08-10T06:29:06,193 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 398a15bfffb84acfadb1be252c2e133a 2023-08-10T06:29:06,194 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/meta/9e97dc38b66b476f952990b23fa0a3ca as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/9e97dc38b66b476f952990b23fa0a3ca 2023-08-10T06:29:06,200 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/9e97dc38b66b476f952990b23fa0a3ca, entries=4, sequenceid=50, filesize=5.5 K 2023-08-10T06:29:06,201 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/session/398a15bfffb84acfadb1be252c2e133a as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/398a15bfffb84acfadb1be252c2e133a 2023-08-10T06:29:06,206 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/398a15bfffb84acfadb1be252c2e133a, entries=2, sequenceid=50, filesize=5.4 K 2023-08-10T06:29:06,207 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1172, heapSize ~2.19 KB/2240, currentSize=0 B/0 for b9c493f2c9fbef45de19e8268a534e72 in 843ms, sequenceid=50, compaction requested=true 2023-08-10T06:29:06,207 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.HRegion(2520): Flush status journal for b9c493f2c9fbef45de19e8268a534e72: 2023-08-10T06:29:06,207 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. for snapshot_backup_system completed. 2023-08-10T06:29:06,207 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72.' region-info for snapshot=snapshot_backup_system 2023-08-10T06:29:06,207 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:29:06,207 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/bc10682e7f36491d941e2fe62e2e4100, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/9d9a4feb1c4e44d99cc63e3f68878731, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/f04a99990b2c429eaf1096ef413be3b8, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/9e97dc38b66b476f952990b23fa0a3ca] hfiles 2023-08-10T06:29:06,207 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (1/4): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/bc10682e7f36491d941e2fe62e2e4100 for snapshot=snapshot_backup_system 2023-08-10T06:29:06,207 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (2/4): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/9d9a4feb1c4e44d99cc63e3f68878731 for snapshot=snapshot_backup_system 2023-08-10T06:29:06,207 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (3/4): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/f04a99990b2c429eaf1096ef413be3b8 for snapshot=snapshot_backup_system 2023-08-10T06:29:06,207 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (4/4): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/9e97dc38b66b476f952990b23fa0a3ca for snapshot=snapshot_backup_system 2023-08-10T06:29:06,208 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/e88678d5b4b14eecabac817daf12539e, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/37561b9a8ae2406bb6d053c33aeff31c, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/883911c19cab473f8bf801f934c90bc1, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/0c7705a45c554d6a97fa37fac319ceff, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/398a15bfffb84acfadb1be252c2e133a] hfiles 2023-08-10T06:29:06,208 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (1/5): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/e88678d5b4b14eecabac817daf12539e for snapshot=snapshot_backup_system 2023-08-10T06:29:06,208 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (2/5): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/37561b9a8ae2406bb6d053c33aeff31c for snapshot=snapshot_backup_system 2023-08-10T06:29:06,208 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (3/5): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/883911c19cab473f8bf801f934c90bc1 for snapshot=snapshot_backup_system 2023-08-10T06:29:06,208 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (4/5): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/0c7705a45c554d6a97fa37fac319ceff for snapshot=snapshot_backup_system 2023-08-10T06:29:06,208 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (5/5): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/398a15bfffb84acfadb1be252c2e133a for snapshot=snapshot_backup_system 2023-08-10T06:29:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=68 2023-08-10T06:29:06,613 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. 2023-08-10T06:29:06,613 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=69 2023-08-10T06:29:06,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster(3994): Remote procedure done, pid=69 2023-08-10T06:29:06,614 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:29:06,615 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=69, ppid=68, state=RUNNABLE; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:29:06,618 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=69, resume processing ppid=68 2023-08-10T06:29:06,618 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:29:06,618 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=69, ppid=68, state=SUCCESS; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72 in 1.4060 sec 2023-08-10T06:29:06,619 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:29:06,619 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:29:06,619 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-08-10T06:29:06,620 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_backup_system 2023-08-10T06:29:06,630 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:29:06,640 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:29:06,640 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_backup_system 2023-08-10T06:29:07,063 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:29:07,064 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 68 2023-08-10T06:29:07,065 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=68, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.8640 sec 2023-08-10T06:29:07,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=68 2023-08-10T06:29:07,308 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(115): Backup backup_1691648945151 started at 1691648947307. 2023-08-10T06:29:07,309 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(119): Backup session backup_1691648945151 has been started. 2023-08-10T06:29:07,314 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-08-10T06:29:07,315 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-08-10T06:29:07,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-08-10T06:29:07,315 INFO [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-08-10T06:29:07,315 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-08-10T06:29:07,316 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-08-10T06:29:07,316 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-08-10T06:29:07,317 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:29:07,319 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:07,319 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:29:07,319 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:29:07,320 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-08-10T06:29:07,320 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-08-10T06:29:07,320 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-08-10T06:29:07,320 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:29:07,320 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-08-10T06:29:07,321 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-08-10T06:29:07,321 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-08-10T06:29:07,321 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-08-10T06:29:07,321 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-08-10T06:29:07,321 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-08-10T06:29:07,321 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-08-10T06:29:07,321 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,45101,1691648859755' joining acquired barrier for procedure (rolllog) in zk 2023-08-10T06:29:07,323 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-08-10T06:29:07,324 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:07,324 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:29:07,324 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:29:07,324 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-08-10T06:29:07,324 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:29:07,324 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:29:07,325 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:29:07,325 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:07,325 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:29:07,325 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,45101,1691648859755' joining acquired barrier for procedure 'rolllog' on coordinator 2023-08-10T06:29:07,325 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@3af13bcd[Count = 0] remaining members to acquire global barrier 2023-08-10T06:29:07,325 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-08-10T06:29:07,325 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-08-10T06:29:07,327 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-08-10T06:29:07,327 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-08-10T06:29:07,327 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-08-10T06:29:07,327 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-08-10T06:29:07,327 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-08-10T06:29:07,327 DEBUG [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:07,327 INFO [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1691648937783 highest: 1691648937783 on jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:07,327 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta:.meta(num 1691648937765) roll requested 2023-08-10T06:29:07,339 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691648947327.meta, exclude list is [], retry=0 2023-08-10T06:29:07,342 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK] 2023-08-10T06:29:07,343 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691648937765.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691648947327.meta 2023-08-10T06:29:07,344 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK]] 2023-08-10T06:29:07,344 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691648937765.meta is not closed yet, will try archiving it next time 2023-08-10T06:29:07,344 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C45101%2C1691648859755:(num 1691648937783) roll requested 2023-08-10T06:29:07,346 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691648937765.meta to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/oldWALs/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691648937765.meta 2023-08-10T06:29:07,356 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691648947344, exclude list is [], retry=0 2023-08-10T06:29:07,359 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK] 2023-08-10T06:29:07,361 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691648937783 with entries=10, filesize=2.31 KB; new WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691648947344 2023-08-10T06:29:07,361 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK]] 2023-08-10T06:29:07,361 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691648937783 is not closed yet, will try archiving it next time 2023-08-10T06:29:07,361 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691648926229 to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/oldWALs/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691648926229 2023-08-10T06:29:07,367 DEBUG [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-08-10T06:29:07,367 INFO [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1691648947344 on jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:07,379 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-08-10T06:29:07,379 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,45101,1691648859755' in zk 2023-08-10T06:29:07,384 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-08-10T06:29:07,384 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:29:07,384 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-08-10T06:29:07,385 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:07,385 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:29:07,385 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:29:07,385 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:29:07,386 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:29:07,386 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:29:07,387 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:07,387 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:29:07,387 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:29:07,388 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:07,392 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,45101,1691648859755': 2023-08-10T06:29:07,392 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,45101,1691648859755' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-08-10T06:29:07,392 INFO [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-08-10T06:29:07,392 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-08-10T06:29:07,392 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-08-10T06:29:07,392 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-08-10T06:29:07,392 INFO [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-08-10T06:29:07,394 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-08-10T06:29:07,394 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:29:07,394 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-08-10T06:29:07,394 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:29:07,394 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:29:07,395 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:29:07,395 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:29:07,395 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:29:07,395 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:29:07,395 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:29:07,395 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:29:07,396 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:29:07,396 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:07,396 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:29:07,397 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:29:07,397 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:07,402 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:29:07,402 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:29:07,402 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:29:07,402 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-08-10T06:29:07,403 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-08-10T06:29:07,403 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:29:07,403 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:29:07,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-08-10T06:29:07,507 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1691648947506_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } 2023-08-10T06:29:07,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691648947507 (current time:1691648947507). 2023-08-10T06:29:07,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:29:07,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1691648947506_default_test-1691648868734 VERSION not specified, setting to 2 2023-08-10T06:29:07,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13fcafbe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:29:07,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:29:07,516 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:58660, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:29:07,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:29:07,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:29:07,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=70, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1691648947506_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } 2023-08-10T06:29:07,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1691648947506_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 }, snapshot procedure id = 70 2023-08-10T06:29:07,520 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1691648947506_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:29:07,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=70 2023-08-10T06:29:07,520 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1691648947506_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:29:07,522 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1691648947506_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:29:07,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=70 2023-08-10T06:29:07,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=70 2023-08-10T06:29:07,933 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1691648947506_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:29:07,934 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE; SnapshotRegionProcedure acd4d10145cd83e849c8f35f15dd289e}] 2023-08-10T06:29:07,935 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=71, ppid=70, state=RUNNABLE; SnapshotRegionProcedure acd4d10145cd83e849c8f35f15dd289e 2023-08-10T06:29:08,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45101 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=71 2023-08-10T06:29:08,087 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1691648868734,,1691648917151.acd4d10145cd83e849c8f35f15dd289e. 2023-08-10T06:29:08,087 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] regionserver.HRegion(2520): Flush status journal for acd4d10145cd83e849c8f35f15dd289e: 2023-08-10T06:29:08,087 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1691648868734,,1691648917151.acd4d10145cd83e849c8f35f15dd289e. for snapshot_1691648947506_default_test-1691648868734 completed. 2023-08-10T06:29:08,088 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] snapshot.SnapshotManifest(240): Storing 'test-1691648868734,,1691648917151.acd4d10145cd83e849c8f35f15dd289e.' region-info for snapshot=snapshot_1691648947506_default_test-1691648868734 2023-08-10T06:29:08,088 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:29:08,088 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/acd4d10145cd83e849c8f35f15dd289e/f/eb3ec537e8f043cfaae1688b3b8e2162_SeqId_4_] hfiles 2023-08-10T06:29:08,088 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/acd4d10145cd83e849c8f35f15dd289e/f/eb3ec537e8f043cfaae1688b3b8e2162_SeqId_4_ for snapshot=snapshot_1691648947506_default_test-1691648868734 2023-08-10T06:29:08,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=70 2023-08-10T06:29:08,494 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1691648868734,,1691648917151.acd4d10145cd83e849c8f35f15dd289e. 2023-08-10T06:29:08,494 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=71 2023-08-10T06:29:08,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster(3994): Remote procedure done, pid=71 2023-08-10T06:29:08,495 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1691648947506_default_test-1691648868734 on region acd4d10145cd83e849c8f35f15dd289e 2023-08-10T06:29:08,496 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=71, ppid=70, state=RUNNABLE; SnapshotRegionProcedure acd4d10145cd83e849c8f35f15dd289e 2023-08-10T06:29:08,498 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=71, resume processing ppid=70 2023-08-10T06:29:08,498 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1691648947506_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:29:08,498 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=71, ppid=70, state=SUCCESS; SnapshotRegionProcedure acd4d10145cd83e849c8f35f15dd289e in 563 msec 2023-08-10T06:29:08,499 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1691648947506_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:29:08,499 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1691648947506_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:29:08,499 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1691648947506_default_test-1691648868734 2023-08-10T06:29:08,500 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_1691648947506_default_test-1691648868734 2023-08-10T06:29:08,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=70 2023-08-10T06:29:08,909 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1691648947506_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:29:08,915 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1691648947506_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:29:08,915 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_1691648947506_default_test-1691648868734 to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691648947506_default_test-1691648868734 2023-08-10T06:29:09,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=70 2023-08-10T06:29:09,733 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1691648947506_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:29:09,733 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1691648947506_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 }, snapshot procedure id = 70 2023-08-10T06:29:09,734 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=70, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1691648947506_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } in 2.2160 sec 2023-08-10T06:29:11,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=70 2023-08-10T06:29:11,627 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1691648945151 2023-08-10T06:29:11,627 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-08-10T06:29:11,627 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-08-10T06:29:11,627 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1691648945151_test-1691648868734 2023-08-10T06:29:11,627 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1691648947506_default_test-1691648868734 to hdfs://localhost:45379/backupUT/backup_1691648945151/default/test-1691648868734/ 2023-08-10T06:29:11,627 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-08-10T06:29:11,650 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:45379, inputRoot=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e 2023-08-10T06:29:11,650 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,33705,1691648856595_1211029290_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:45379/backupUT/backup_1691648945151/default/test-1691648868734, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:45379/backupUT/backup_1691648945151/default/test-1691648868734/.hbase-snapshot/.tmp/snapshot_1691648947506_default_test-1691648868734 2023-08-10T06:29:11,650 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:45379, inputRoot=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e, snapshotDir=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691648947506_default_test-1691648868734. 2023-08-10T06:29:11,658 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691648947506_default_test-1691648868734 to hdfs://localhost:45379/backupUT/backup_1691648945151/default/test-1691648868734/.hbase-snapshot/.tmp/snapshot_1691648947506_default_test-1691648868734 2023-08-10T06:29:12,078 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:12,078 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:12,079 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:12,079 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:13,107 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/hadoop-5206731560755191351.jar 2023-08-10T06:29:13,107 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:13,108 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:13,108 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:13,109 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:13,109 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:13,109 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:13,110 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:13,110 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:13,110 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:13,111 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:29:13,112 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-08-10T06:29:13,112 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-08-10T06:29:13,112 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-08-10T06:29:13,113 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-08-10T06:29:13,113 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-08-10T06:29:13,114 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-08-10T06:29:13,114 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-08-10T06:29:13,115 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-08-10T06:29:13,115 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-08-10T06:29:13,115 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-08-10T06:29:13,116 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-08-10T06:29:13,116 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:29:13,117 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:29:13,118 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:29:13,118 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:29:13,118 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:29:13,119 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:29:13,119 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:29:13,161 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1691648947506_default_test-1691648868734' hfile list 2023-08-10T06:29:13,164 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-08-10T06:29:13,232 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691648953198/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-08-10T06:29:13,232 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691648953198/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-08-10T06:29:13,287 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-08-10T06:29:13,311 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/acd4d10145cd83e849c8f35f15dd289e/f/eb3ec537e8f043cfaae1688b3b8e2162_SeqId_4_ output=hdfs://localhost:45379/backupUT/backup_1691648945151/default/test-1691648868734/archive/data/default/test-1691648868734/acd4d10145cd83e849c8f35f15dd289e/f/eb3ec537e8f043cfaae1688b3b8e2162_SeqId_4_ 2023-08-10T06:29:13,311 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8724 (8.5 K) time=0sec 4.160M/sec 2023-08-10T06:29:14,269 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-08-10T06:29:14,270 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-08-10T06:29:14,276 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1691648947506_default_test-1691648868734 2023-08-10T06:29:14,277 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1691648947506_default_test-1691648868734 finished. 2023-08-10T06:29:14,277 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(1014): test-1691648868734 2023-08-10T06:29:14,283 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691648945151 2023-08-10T06:29:14,283 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-08-10T06:29:14,688 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:45379/backupUT/backup_1691648945151/.backup.manifest 2023-08-10T06:29:14,688 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(358): Backup backup_1691648945151 finished: type=FULL,tablelist=test-1691648868734,targetRootDir=hdfs://localhost:45379/backupUT,startts=1691648947307,completets=1691648954283,bytescopied=0 2023-08-10T06:29:14,688 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-08-10T06:29:14,688 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1691648947506_default_test-1691648868734 2023-08-10T06:29:14,689 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1691648947506_default_test-1691648868734" 2023-08-10T06:29:14,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1691648947506_default_test-1691648868734 2023-08-10T06:29:14,692 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1691648947506_default_test-1691648868734 for backup backup_1691648945151 succeeded. 2023-08-10T06:29:14,693 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-08-10T06:29:14,698 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-08-10T06:29:14,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-08-10T06:29:14,700 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-08-10T06:29:14,702 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-08-10T06:29:14,707 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(376): Backup backup_1691648945151 completed. 2023-08-10T06:29:14,707 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:29:14,718 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] backup.TestFullRestore(61): backup complete 2023-08-10T06:29:14,719 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(101): HBase table table1 does not exist. It will be created during restore process 2023-08-10T06:29:14,720 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:45379/backupUT/backup_1691648945151/.backup.manifest 2023-08-10T06:29:14,721 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691648945151/.backup.manifest 2023-08-10T06:29:14,722 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:45379/backupUT/backup_1691648945151/.backup.manifest 2023-08-10T06:29:14,723 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691648945151/.backup.manifest 2023-08-10T06:29:14,724 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(148): Restoring 'test-1691648868734' to 'table1' from full backup image hdfs://localhost:45379/backupUT/backup_1691648945151/default/test-1691648868734 2023-08-10T06:29:14,730 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] util.RestoreTool(487): Creating target table 'table1' 2023-08-10T06:29:14,730 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] util.RestoreTool(391): Parsing region dir: hdfs://localhost:45379/backupUT/backup_1691648945151/default/test-1691648868734/archive/data/default/test-1691648868734/acd4d10145cd83e849c8f35f15dd289e 2023-08-10T06:29:14,731 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:45379/backupUT/backup_1691648945151/default/test-1691648868734/archive/data/default/test-1691648868734/acd4d10145cd83e849c8f35f15dd289e/f in region [hdfs://localhost:45379/backupUT/backup_1691648945151/default/test-1691648868734/archive/data/default/test-1691648868734/acd4d10145cd83e849c8f35f15dd289e] 2023-08-10T06:29:14,734 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] util.RestoreTool(442): Trying to figure out region boundaries hfile=hdfs://localhost:45379/backupUT/backup_1691648945151/default/test-1691648868734/archive/data/default/test-1691648868734/acd4d10145cd83e849c8f35f15dd289e/f/eb3ec537e8f043cfaae1688b3b8e2162_SeqId_4_ first=row0 last=row98 2023-08-10T06:29:14,734 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: eb3ec537e8f043cfaae1688b3b8e2162_SeqId_4_ 2023-08-10T06:29:14,736 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:29:14,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=72, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=table1 2023-08-10T06:29:14,739 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_PRE_OPERATION 2023-08-10T06:29:14,739 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "table1" procId is: 72 2023-08-10T06:29:14,739 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-08-10T06:29:14,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=72 2023-08-10T06:29:14,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=72 2023-08-10T06:29:15,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=72 2023-08-10T06:29:15,147 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => bb9a13d518c7822a0283ad492a881a0c, NAME => 'table1,,1691648954736.bb9a13d518c7822a0283ad492a881a0c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e 2023-08-10T06:29:15,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=72 2023-08-10T06:29:15,553 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(888): Instantiated table1,,1691648954736.bb9a13d518c7822a0283ad492a881a0c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:29:15,553 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1668): Closing bb9a13d518c7822a0283ad492a881a0c, disabling compactions & flushes 2023-08-10T06:29:15,553 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1690): Closing region table1,,1691648954736.bb9a13d518c7822a0283ad492a881a0c. 2023-08-10T06:29:15,553 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on table1,,1691648954736.bb9a13d518c7822a0283ad492a881a0c. 2023-08-10T06:29:15,553 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on table1,,1691648954736.bb9a13d518c7822a0283ad492a881a0c. after waiting 0 ms 2023-08-10T06:29:15,553 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region table1,,1691648954736.bb9a13d518c7822a0283ad492a881a0c. 2023-08-10T06:29:15,553 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1909): Closed table1,,1691648954736.bb9a13d518c7822a0283ad492a881a0c. 2023-08-10T06:29:15,553 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1622): Region close journal for bb9a13d518c7822a0283ad492a881a0c: 2023-08-10T06:29:15,554 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ADD_TO_META 2023-08-10T06:29:15,554 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"table1,,1691648954736.bb9a13d518c7822a0283ad492a881a0c.","families":{"info":[{"qualifier":"regioninfo","vlen":40,"tag":[],"timestamp":"1691648955554"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691648955554"}]},"ts":"1691648955554"} 2023-08-10T06:29:15,556 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:29:15,556 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-08-10T06:29:15,556 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648955556"}]},"ts":"1691648955556"} 2023-08-10T06:29:15,557 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=ENABLING in hbase:meta 2023-08-10T06:29:15,563 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=table1, region=bb9a13d518c7822a0283ad492a881a0c, ASSIGN}] 2023-08-10T06:29:15,564 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=table1, region=bb9a13d518c7822a0283ad492a881a0c, ASSIGN 2023-08-10T06:29:15,565 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=table1, region=bb9a13d518c7822a0283ad492a881a0c, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45101,1691648859755; forceNewPlan=false, retain=false 2023-08-10T06:29:15,716 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=73 updating hbase:meta row=bb9a13d518c7822a0283ad492a881a0c, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:15,718 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE; OpenRegionProcedure bb9a13d518c7822a0283ad492a881a0c, server=jenkins-hbase3.apache.org,45101,1691648859755}] 2023-08-10T06:29:15,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=72 2023-08-10T06:29:15,873 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(130): Open table1,,1691648954736.bb9a13d518c7822a0283ad492a881a0c. 2023-08-10T06:29:15,873 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7254): Opening region: {ENCODED => bb9a13d518c7822a0283ad492a881a0c, NAME => 'table1,,1691648954736.bb9a13d518c7822a0283ad492a881a0c.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:29:15,873 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:29:15,874 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 bb9a13d518c7822a0283ad492a881a0c 2023-08-10T06:29:15,874 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(888): Instantiated table1,,1691648954736.bb9a13d518c7822a0283ad492a881a0c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:29:15,874 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7296): checking encryption for bb9a13d518c7822a0283ad492a881a0c 2023-08-10T06:29:15,874 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7299): checking classloading for bb9a13d518c7822a0283ad492a881a0c 2023-08-10T06:29:15,875 INFO [StoreOpener-bb9a13d518c7822a0283ad492a881a0c-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region bb9a13d518c7822a0283ad492a881a0c 2023-08-10T06:29:15,876 INFO [StoreOpener-bb9a13d518c7822a0283ad492a881a0c-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bb9a13d518c7822a0283ad492a881a0c columnFamilyName f 2023-08-10T06:29:15,876 DEBUG [StoreOpener-bb9a13d518c7822a0283ad492a881a0c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:29:15,877 INFO [StoreOpener-bb9a13d518c7822a0283ad492a881a0c-1 {}] regionserver.HStore(324): Store=bb9a13d518c7822a0283ad492a881a0c/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:29:15,878 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/table1/bb9a13d518c7822a0283ad492a881a0c 2023-08-10T06:29:15,878 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/table1/bb9a13d518c7822a0283ad492a881a0c 2023-08-10T06:29:15,880 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1076): writing seq id for bb9a13d518c7822a0283ad492a881a0c 2023-08-10T06:29:15,882 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/table1/bb9a13d518c7822a0283ad492a881a0c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:29:15,882 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1093): Opened bb9a13d518c7822a0283ad492a881a0c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11823549280, jitterRate=0.10115383565425873}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-08-10T06:29:15,883 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(992): Region open journal for bb9a13d518c7822a0283ad492a881a0c: 2023-08-10T06:29:15,883 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2552): Post open deploy tasks for table1,,1691648954736.bb9a13d518c7822a0283ad492a881a0c., pid=74, masterSystemTime=1691648955870 2023-08-10T06:29:15,885 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2579): Finished post open deploy task for table1,,1691648954736.bb9a13d518c7822a0283ad492a881a0c. 2023-08-10T06:29:15,885 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(158): Opened table1,,1691648954736.bb9a13d518c7822a0283ad492a881a0c. 2023-08-10T06:29:15,885 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=73 updating hbase:meta row=bb9a13d518c7822a0283ad492a881a0c, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:15,888 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=74, resume processing ppid=73 2023-08-10T06:29:15,888 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=74, ppid=73, state=SUCCESS; OpenRegionProcedure bb9a13d518c7822a0283ad492a881a0c, server=jenkins-hbase3.apache.org,45101,1691648859755 in 168 msec 2023-08-10T06:29:15,889 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=73, resume processing ppid=72 2023-08-10T06:29:15,889 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=73, ppid=72, state=SUCCESS; TransitRegionStateProcedure table=table1, region=bb9a13d518c7822a0283ad492a881a0c, ASSIGN in 325 msec 2023-08-10T06:29:15,889 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-08-10T06:29:15,889 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648955889"}]},"ts":"1691648955889"} 2023-08-10T06:29:15,890 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=ENABLED in hbase:meta 2023-08-10T06:29:15,892 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_POST_OPERATION 2023-08-10T06:29:15,894 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=72, state=SUCCESS; CreateTableProcedure table=table1 in 1.1560 sec 2023-08-10T06:29:16,762 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_table1 2023-08-10T06:29:16,762 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_table1 Metrics about Tables on a single HBase RegionServer 2023-08-10T06:29:16,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=72 2023-08-10T06:29:16,848 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:45379/backupUT/backup_1691648945151/default/test-1691648868734/archive/data/default/test-1691648868734/acd4d10145cd83e849c8f35f15dd289e from hbase tables test-1691648868734 to tables table1 2023-08-10T06:29:16,848 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceRestoreJob(72): Restore test-1691648868734 into table1 2023-08-10T06:29:16,850 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-default-table1-1691648956848 from hdfs://localhost:45379/backupUT/backup_1691648945151/default/test-1691648868734/archive/data/default/test-1691648868734/acd4d10145cd83e849c8f35f15dd289e 2023-08-10T06:29:16,855 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d9a4983, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:29:16,856 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:29:16,857 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:49832, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:29:16,858 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-08-10T06:29:16,858 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table table1 2023-08-10T06:29:16,860 DEBUG [hconnection-0x60c095dc-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:29:16,862 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:52712, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:29:16,863 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for table1: 2023-08-10T06:29:16,863 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-08-10T06:29:16,864 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/partitions_02ca92ce-aaeb-42c2-a772-f7366a8ba219 2023-08-10T06:29:17,270 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:17,271 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:17,271 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:17,271 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:18,253 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/hadoop-4907320178590439215.jar 2023-08-10T06:29:18,254 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:18,254 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:18,254 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:18,255 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:18,255 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:18,255 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:18,256 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:18,256 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:18,256 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:18,257 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:29:18,258 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-08-10T06:29:18,258 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-08-10T06:29:18,258 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-08-10T06:29:18,259 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-08-10T06:29:18,259 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-08-10T06:29:18,260 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-08-10T06:29:18,260 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-08-10T06:29:18,260 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-08-10T06:29:18,261 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-08-10T06:29:18,261 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-08-10T06:29:18,262 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-08-10T06:29:18,262 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:18,263 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:18,263 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:18,263 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:18,263 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:18,264 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:18,264 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:29:18,265 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: table1 2023-08-10T06:29:18,265 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:29:18,265 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-08-10T06:29:18,265 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-08-10T06:29:18,266 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:29:18,277 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-08-10T06:29:18,367 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691648958332/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-08-10T06:29:18,367 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691648958332/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-08-10T06:29:18,415 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:45379/backupUT/backup_1691648945151/default/test-1691648868734/archive/data/default/test-1691648868734/acd4d10145cd83e849c8f35f15dd289e/f/eb3ec537e8f043cfaae1688b3b8e2162_SeqId_4_ 2023-08-10T06:29:18,419 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-08-10T06:29:18,422 DEBUG [LocalJobRunner Map Task Executor #0 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: eb3ec537e8f043cfaae1688b3b8e2162_SeqId_4_ 2023-08-10T06:29:18,441 DEBUG [pool-297-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32eb7fcd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:29:18,442 DEBUG [hconnection-0xad5a0df-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:29:18,443 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:52728, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:29:18,445 DEBUG [pool-297-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:29:18,446 DEBUG [pool-297-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-08-10T06:29:18,446 DEBUG [pool-297-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-08-10T06:29:18,453 DEBUG [pool-297-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-default-table1-1691648956848/_temporary/0/_temporary/attempt_local402876691_0009_r_000000_0/f/06fac9f71f59474f8f799050b67c5113 is 37, key is row10/f:q1/1691648871386/Put/seqid=0 2023-08-10T06:29:18,454 WARN [Thread-1345 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:45101] Specified favored nodes: [jenkins-hbase3.apache.org:45101] 2023-08-10T06:29:18,548 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'table1' 2023-08-10T06:29:19,414 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-default-table1-1691648956848 2023-08-10T06:29:19,419 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19a7a4fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:29:19,421 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:29:19,423 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:52732, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:29:19,429 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-default-table1-1691648956848/_SUCCESS 2023-08-10T06:29:19,433 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:29:19,434 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:49840, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:29:19,444 DEBUG [LoadIncrementalHFiles-0 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 06fac9f71f59474f8f799050b67c5113 2023-08-10T06:29:19,444 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-default-table1-1691648956848/f/06fac9f71f59474f8f799050b67c5113 first=Optional[row0] last=Optional[row98] 2023-08-10T06:29:19,445 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=table1,,1691648954736.bb9a13d518c7822a0283ad492a881a0c., hostname=jenkins-hbase3.apache.org,45101,1691648859755, seqNum=2 for row with hfile group [{f,hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-default-table1-1691648956848/f/06fac9f71f59474f8f799050b67c5113}] 2023-08-10T06:29:19,452 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-default-table1-1691648956848/f/06fac9f71f59474f8f799050b67c5113 for inclusion in bb9a13d518c7822a0283ad492a881a0c/f 2023-08-10T06:29:19,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-08-10T06:29:19,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HStore(625): Region bounds: first= last= 2023-08-10T06:29:19,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 06fac9f71f59474f8f799050b67c5113 2023-08-10T06:29:19,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HRegion(2520): Flush status journal for bb9a13d518c7822a0283ad492a881a0c: 2023-08-10T06:29:19,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-default-table1-1691648956848/f/06fac9f71f59474f8f799050b67c5113 to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/staging/jenkins__table1__u4nofdum9nc3031ba1mmo7ql2ih2ju4pn9ru1cvmpolh8k74bt69t04qpl7t68gu/f/06fac9f71f59474f8f799050b67c5113 2023-08-10T06:29:19,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/staging/jenkins__table1__u4nofdum9nc3031ba1mmo7ql2ih2ju4pn9ru1cvmpolh8k74bt69t04qpl7t68gu/f/06fac9f71f59474f8f799050b67c5113 as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/table1/bb9a13d518c7822a0283ad492a881a0c/f/9d7737ce32e94140b9cb81af0d3325b0_SeqId_4_ 2023-08-10T06:29:19,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dfcfb7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:29:19,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:29:19,472 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:49852, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:29:19,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:29:19,505 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:52744, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:29:19,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:29:19,519 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/staging/jenkins__table1__u4nofdum9nc3031ba1mmo7ql2ih2ju4pn9ru1cvmpolh8k74bt69t04qpl7t68gu/f/06fac9f71f59474f8f799050b67c5113 into bb9a13d518c7822a0283ad492a881a0c/f as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/table1/bb9a13d518c7822a0283ad492a881a0c/f/9d7737ce32e94140b9cb81af0d3325b0_SeqId_4_ - updating store file list. 2023-08-10T06:29:19,524 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/table1/bb9a13d518c7822a0283ad492a881a0c/f/9d7737ce32e94140b9cb81af0d3325b0_SeqId_4_ into bb9a13d518c7822a0283ad492a881a0c/f 2023-08-10T06:29:19,524 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/staging/jenkins__table1__u4nofdum9nc3031ba1mmo7ql2ih2ju4pn9ru1cvmpolh8k74bt69t04qpl7t68gu/f/06fac9f71f59474f8f799050b67c5113 into bb9a13d518c7822a0283ad492a881a0c/f (new location: hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/table1/bb9a13d518c7822a0283ad492a881a0c/f/9d7737ce32e94140b9cb81af0d3325b0_SeqId_4_) 2023-08-10T06:29:19,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/staging/jenkins__table1__u4nofdum9nc3031ba1mmo7ql2ih2ju4pn9ru1cvmpolh8k74bt69t04qpl7t68gu/f/06fac9f71f59474f8f799050b67c5113 2023-08-10T06:29:19,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@330ed639, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:29:19,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:29:19,531 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:49864, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-08-10T06:29:19,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:29:19,535 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:52750, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-08-10T06:29:19,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:29:19,547 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:29:19,547 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-08-10T06:29:19,547 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-08-10T06:29:19,547 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(233): Backup: backup_1691648945151 hdfs://localhost:45379/backupUT/backup_1691648945151/default/test-1691648868734/ 2023-08-10T06:29:19,547 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-08-10T06:29:19,549 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster$12(2663): Client=jenkins//172.31.12.81 disable table1 2023-08-10T06:29:19,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=75, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=table1 2023-08-10T06:29:19,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=75 2023-08-10T06:29:19,552 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648959552"}]},"ts":"1691648959552"} 2023-08-10T06:29:19,553 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=DISABLING in hbase:meta 2023-08-10T06:29:19,556 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(293): Set table1 to state=DISABLING 2023-08-10T06:29:19,557 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=table1, region=bb9a13d518c7822a0283ad492a881a0c, UNASSIGN}] 2023-08-10T06:29:19,558 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=table1, region=bb9a13d518c7822a0283ad492a881a0c, UNASSIGN 2023-08-10T06:29:19,558 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=76 updating hbase:meta row=bb9a13d518c7822a0283ad492a881a0c, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:19,559 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE; CloseRegionProcedure bb9a13d518c7822a0283ad492a881a0c, server=jenkins-hbase3.apache.org,45101,1691648859755}] 2023-08-10T06:29:19,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=75 2023-08-10T06:29:19,711 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(119): Close bb9a13d518c7822a0283ad492a881a0c 2023-08-10T06:29:19,711 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1668): Closing bb9a13d518c7822a0283ad492a881a0c, disabling compactions & flushes 2023-08-10T06:29:19,711 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1690): Closing region table1,,1691648954736.bb9a13d518c7822a0283ad492a881a0c. 2023-08-10T06:29:19,711 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1711): Waiting without time limit for close lock on table1,,1691648954736.bb9a13d518c7822a0283ad492a881a0c. 2023-08-10T06:29:19,711 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1778): Acquired close lock on table1,,1691648954736.bb9a13d518c7822a0283ad492a881a0c. after waiting 0 ms 2023-08-10T06:29:19,711 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1788): Updates disabled for region table1,,1691648954736.bb9a13d518c7822a0283ad492a881a0c. 2023-08-10T06:29:19,713 DEBUG [StoreFileCloser-bb9a13d518c7822a0283ad492a881a0c-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 9d7737ce32e94140b9cb81af0d3325b0_SeqId_4_ 2023-08-10T06:29:19,717 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/table1/bb9a13d518c7822a0283ad492a881a0c/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2023-08-10T06:29:19,718 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:29:19,718 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1909): Closed table1,,1691648954736.bb9a13d518c7822a0283ad492a881a0c. 2023-08-10T06:29:19,718 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1622): Region close journal for bb9a13d518c7822a0283ad492a881a0c: 2023-08-10T06:29:19,719 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(163): Closed bb9a13d518c7822a0283ad492a881a0c 2023-08-10T06:29:19,719 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=76 updating hbase:meta row=bb9a13d518c7822a0283ad492a881a0c, regionState=CLOSED 2023-08-10T06:29:19,722 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=77, resume processing ppid=76 2023-08-10T06:29:19,722 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=77, ppid=76, state=SUCCESS; CloseRegionProcedure bb9a13d518c7822a0283ad492a881a0c, server=jenkins-hbase3.apache.org,45101,1691648859755 in 161 msec 2023-08-10T06:29:19,723 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=76, resume processing ppid=75 2023-08-10T06:29:19,723 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=76, ppid=75, state=SUCCESS; TransitRegionStateProcedure table=table1, region=bb9a13d518c7822a0283ad492a881a0c, UNASSIGN in 165 msec 2023-08-10T06:29:19,724 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648959724"}]},"ts":"1691648959724"} 2023-08-10T06:29:19,725 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=DISABLED in hbase:meta 2023-08-10T06:29:19,727 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(305): Set table1 to state=DISABLED 2023-08-10T06:29:19,728 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=75, state=SUCCESS; DisableTableProcedure table=table1 in 178 msec 2023-08-10T06:29:19,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=75 2023-08-10T06:29:19,859 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster$5(2447): Client=jenkins//172.31.12.81 delete table1 2023-08-10T06:29:19,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=78, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=table1 2023-08-10T06:29:19,869 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=78, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=table1 2023-08-10T06:29:19,869 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=78, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=table1 2023-08-10T06:29:19,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=78 2023-08-10T06:29:19,871 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/table1/bb9a13d518c7822a0283ad492a881a0c 2023-08-10T06:29:19,873 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/table1/bb9a13d518c7822a0283ad492a881a0c/f, FileablePath, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/table1/bb9a13d518c7822a0283ad492a881a0c/recovered.edits] 2023-08-10T06:29:19,877 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/table1/bb9a13d518c7822a0283ad492a881a0c/f/9d7737ce32e94140b9cb81af0d3325b0_SeqId_4_ to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/archive/data/default/table1/bb9a13d518c7822a0283ad492a881a0c/f/9d7737ce32e94140b9cb81af0d3325b0_SeqId_4_ 2023-08-10T06:29:19,880 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/table1/bb9a13d518c7822a0283ad492a881a0c/recovered.edits/6.seqid to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/archive/data/default/table1/bb9a13d518c7822a0283ad492a881a0c/recovered.edits/6.seqid 2023-08-10T06:29:19,880 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/table1/bb9a13d518c7822a0283ad492a881a0c 2023-08-10T06:29:19,880 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(293): Archived table1 regions 2023-08-10T06:29:19,882 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=78, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=table1 2023-08-10T06:29:19,884 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of table1 from hbase:meta 2023-08-10T06:29:19,886 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(387): Removing 'table1' descriptor. 2023-08-10T06:29:19,887 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=78, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=table1 2023-08-10T06:29:19,887 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(377): Removing 'table1' from region states. 2023-08-10T06:29:19,887 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"table1,,1691648954736.bb9a13d518c7822a0283ad492a881a0c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1691648959887"}]},"ts":"9223372036854775807"} 2023-08-10T06:29:19,889 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-08-10T06:29:19,889 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => bb9a13d518c7822a0283ad492a881a0c, NAME => 'table1,,1691648954736.bb9a13d518c7822a0283ad492a881a0c.', STARTKEY => '', ENDKEY => ''}] 2023-08-10T06:29:19,889 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(381): Marking 'table1' as deleted. 2023-08-10T06:29:19,889 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1691648959889"}]},"ts":"9223372036854775807"} 2023-08-10T06:29:19,890 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1678): Deleted table table1 state from META 2023-08-10T06:29:19,892 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(127): Finished pid=78, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=table1 2023-08-10T06:29:19,893 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=78, state=SUCCESS; DeleteTableProcedure table=table1 in 32 msec 2023-08-10T06:29:19,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=78 2023-08-10T06:29:19,989 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreSingle Thread=594 (was 595), OpenFileDescriptor=670 (was 680), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=126 (was 154), ProcessCount=170 (was 171), AvailableMemoryMB=1584 (was 1965) 2023-08-10T06:29:19,989 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(130): Thread=594 is superior to 500 2023-08-10T06:29:20,004 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreSingleOverwriteCommand Thread=594, OpenFileDescriptor=670, MaxFileDescriptor=60000, SystemLoadAverage=126, ProcessCount=170, AvailableMemoryMB=1583 2023-08-10T06:29:20,004 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(130): Thread=594 is superior to 500 2023-08-10T06:29:20,004 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] backup.TestFullRestore(192): test full restore on a single table empty table: command-line 2023-08-10T06:29:20,009 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@315a9ad1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:29:20,012 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:29:20,014 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:52756, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:29:20,015 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:29:20,017 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:49874, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:29:20,033 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-08-10T06:29:20,038 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:29:20,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691648960038 (current time:1691648960038). 2023-08-10T06:29:20,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:29:20,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-08-10T06:29:20,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a68c098, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:29:20,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:29:20,047 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:52770, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:29:20,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:29:20,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:29:20,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=79, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:29:20,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 79 2023-08-10T06:29:20,051 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:29:20,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=79 2023-08-10T06:29:20,051 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:29:20,053 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:29:20,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=79 2023-08-10T06:29:20,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=79 2023-08-10T06:29:20,462 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:29:20,462 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72}] 2023-08-10T06:29:20,464 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=80, ppid=79, state=RUNNABLE; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:29:20,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45101 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=80 2023-08-10T06:29:20,616 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. 2023-08-10T06:29:20,616 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.HRegion(2819): Flushing b9c493f2c9fbef45de19e8268a534e72 2/2 column families, dataSize=1.14 KB heapSize=2.22 KB 2023-08-10T06:29:20,620 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/meta/930c578fe3f543e1b6308abfb31c7246 is 169, key is trslm:hdfs://localhost:45379/backupUT\x00test-1691648868734/meta:log-roll-map/1691648954279/Put/seqid=0 2023-08-10T06:29:20,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=79 2023-08-10T06:29:21,024 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=451 B at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/meta/930c578fe3f543e1b6308abfb31c7246 2023-08-10T06:29:21,030 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 930c578fe3f543e1b6308abfb31c7246 2023-08-10T06:29:21,034 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/session/adceb6320b02451abd02beed6c69c000 is 452, key is session:backup_1691648945151/session:context/1691648954701/Put/seqid=0 2023-08-10T06:29:21,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=79 2023-08-10T06:29:21,430 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-08-10T06:29:21,439 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=721 B at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/session/adceb6320b02451abd02beed6c69c000 2023-08-10T06:29:21,446 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: adceb6320b02451abd02beed6c69c000 2023-08-10T06:29:21,447 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/meta/930c578fe3f543e1b6308abfb31c7246 as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/930c578fe3f543e1b6308abfb31c7246 2023-08-10T06:29:21,453 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/930c578fe3f543e1b6308abfb31c7246, entries=4, sequenceid=61, filesize=5.5 K 2023-08-10T06:29:21,454 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/session/adceb6320b02451abd02beed6c69c000 as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/adceb6320b02451abd02beed6c69c000 2023-08-10T06:29:21,460 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/adceb6320b02451abd02beed6c69c000, entries=2, sequenceid=61, filesize=5.4 K 2023-08-10T06:29:21,461 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1172, heapSize ~2.19 KB/2240, currentSize=0 B/0 for b9c493f2c9fbef45de19e8268a534e72 in 845ms, sequenceid=61, compaction requested=true 2023-08-10T06:29:21,462 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.HRegion(2520): Flush status journal for b9c493f2c9fbef45de19e8268a534e72: 2023-08-10T06:29:21,462 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. for snapshot_backup_system completed. 2023-08-10T06:29:21,462 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72.' region-info for snapshot=snapshot_backup_system 2023-08-10T06:29:21,462 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:29:21,462 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/bc10682e7f36491d941e2fe62e2e4100, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/9d9a4feb1c4e44d99cc63e3f68878731, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/f04a99990b2c429eaf1096ef413be3b8, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/9e97dc38b66b476f952990b23fa0a3ca, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/930c578fe3f543e1b6308abfb31c7246] hfiles 2023-08-10T06:29:21,462 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (1/5): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/bc10682e7f36491d941e2fe62e2e4100 for snapshot=snapshot_backup_system 2023-08-10T06:29:21,462 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (2/5): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/9d9a4feb1c4e44d99cc63e3f68878731 for snapshot=snapshot_backup_system 2023-08-10T06:29:21,462 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (3/5): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/f04a99990b2c429eaf1096ef413be3b8 for snapshot=snapshot_backup_system 2023-08-10T06:29:21,462 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (4/5): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/9e97dc38b66b476f952990b23fa0a3ca for snapshot=snapshot_backup_system 2023-08-10T06:29:21,462 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (5/5): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/930c578fe3f543e1b6308abfb31c7246 for snapshot=snapshot_backup_system 2023-08-10T06:29:21,462 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/e88678d5b4b14eecabac817daf12539e, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/37561b9a8ae2406bb6d053c33aeff31c, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/883911c19cab473f8bf801f934c90bc1, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/0c7705a45c554d6a97fa37fac319ceff, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/398a15bfffb84acfadb1be252c2e133a, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/adceb6320b02451abd02beed6c69c000] hfiles 2023-08-10T06:29:21,462 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (1/6): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/e88678d5b4b14eecabac817daf12539e for snapshot=snapshot_backup_system 2023-08-10T06:29:21,462 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (2/6): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/37561b9a8ae2406bb6d053c33aeff31c for snapshot=snapshot_backup_system 2023-08-10T06:29:21,462 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (3/6): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/883911c19cab473f8bf801f934c90bc1 for snapshot=snapshot_backup_system 2023-08-10T06:29:21,463 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (4/6): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/0c7705a45c554d6a97fa37fac319ceff for snapshot=snapshot_backup_system 2023-08-10T06:29:21,463 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (5/6): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/398a15bfffb84acfadb1be252c2e133a for snapshot=snapshot_backup_system 2023-08-10T06:29:21,463 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (6/6): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/adceb6320b02451abd02beed6c69c000 for snapshot=snapshot_backup_system 2023-08-10T06:29:21,872 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. 2023-08-10T06:29:21,872 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=80 2023-08-10T06:29:21,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster(3994): Remote procedure done, pid=80 2023-08-10T06:29:21,873 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:29:21,874 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=80, ppid=79, state=RUNNABLE; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:29:21,876 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=80, resume processing ppid=79 2023-08-10T06:29:21,877 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:29:21,877 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=80, ppid=79, state=SUCCESS; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72 in 1.4130 sec 2023-08-10T06:29:21,877 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:29:21,878 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:29:21,878 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-08-10T06:29:21,879 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_backup_system 2023-08-10T06:29:22,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=79 2023-08-10T06:29:22,291 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:29:22,304 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:29:22,304 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_backup_system 2023-08-10T06:29:23,128 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:29:23,128 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 79 2023-08-10T06:29:23,129 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=79, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 3.0800 sec 2023-08-10T06:29:24,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=79 2023-08-10T06:29:24,158 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(115): Backup backup_1691648960009 started at 1691648964157. 2023-08-10T06:29:24,160 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(119): Backup session backup_1691648960009 has been started. 2023-08-10T06:29:24,165 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-08-10T06:29:24,165 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-08-10T06:29:24,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-08-10T06:29:24,166 INFO [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-08-10T06:29:24,166 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-08-10T06:29:24,167 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-08-10T06:29:24,167 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-08-10T06:29:24,167 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:29:24,172 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:29:24,172 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:29:24,172 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:24,172 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-08-10T06:29:24,172 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-08-10T06:29:24,173 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-08-10T06:29:24,173 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:29:24,173 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-08-10T06:29:24,173 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-08-10T06:29:24,173 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-08-10T06:29:24,173 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-08-10T06:29:24,173 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-08-10T06:29:24,174 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-08-10T06:29:24,174 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-08-10T06:29:24,174 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,45101,1691648859755' joining acquired barrier for procedure (rolllog) in zk 2023-08-10T06:29:24,176 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-08-10T06:29:24,176 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:24,176 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:29:24,177 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:29:24,177 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-08-10T06:29:24,177 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:29:24,177 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:29:24,177 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:29:24,178 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:24,178 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:29:24,178 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,45101,1691648859755' joining acquired barrier for procedure 'rolllog' on coordinator 2023-08-10T06:29:24,179 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@39b0b401[Count = 0] remaining members to acquire global barrier 2023-08-10T06:29:24,179 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-08-10T06:29:24,179 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-08-10T06:29:24,181 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-08-10T06:29:24,181 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-08-10T06:29:24,181 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-08-10T06:29:24,181 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-08-10T06:29:24,181 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-08-10T06:29:24,181 DEBUG [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:24,181 INFO [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1691648947344 highest: 1691648947344 on jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:24,181 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta:.meta(num 1691648947327) roll requested 2023-08-10T06:29:24,195 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691648964181.meta, exclude list is [], retry=0 2023-08-10T06:29:24,202 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK] 2023-08-10T06:29:24,205 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691648947327.meta with entries=12, filesize=2.73 KB; new WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691648964181.meta 2023-08-10T06:29:24,205 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK]] 2023-08-10T06:29:24,205 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691648947327.meta is not closed yet, will try archiving it next time 2023-08-10T06:29:24,205 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C45101%2C1691648859755:(num 1691648947344) roll requested 2023-08-10T06:29:24,217 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691648964205, exclude list is [], retry=0 2023-08-10T06:29:24,220 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK] 2023-08-10T06:29:24,221 DEBUG [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-08-10T06:29:24,221 INFO [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1691648964205 on jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:24,222 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691648947344 with entries=13, filesize=3.09 KB; new WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691648964205 2023-08-10T06:29:24,222 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK]] 2023-08-10T06:29:24,222 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691648947344 is not closed yet, will try archiving it next time 2023-08-10T06:29:24,222 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691648937783 to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/oldWALs/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691648937783 2023-08-10T06:29:24,234 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-08-10T06:29:24,234 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,45101,1691648859755' in zk 2023-08-10T06:29:24,236 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-08-10T06:29:24,236 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:24,236 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:29:24,236 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:29:24,236 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-08-10T06:29:24,237 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:29:24,237 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:29:24,237 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:29:24,238 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:29:24,238 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:24,238 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:29:24,238 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:29:24,239 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:24,239 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,45101,1691648859755': 2023-08-10T06:29:24,239 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,45101,1691648859755' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-08-10T06:29:24,239 INFO [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-08-10T06:29:24,239 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-08-10T06:29:24,239 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-08-10T06:29:24,239 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-08-10T06:29:24,239 INFO [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-08-10T06:29:24,241 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-08-10T06:29:24,241 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-08-10T06:29:24,241 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:29:24,241 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:29:24,241 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:29:24,241 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:29:24,241 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:29:24,241 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:29:24,242 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:29:24,242 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:29:24,242 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:29:24,243 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:29:24,243 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:24,243 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:29:24,243 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:29:24,244 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:24,247 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:29:24,247 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:29:24,247 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:29:24,247 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-08-10T06:29:24,248 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-08-10T06:29:24,248 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:29:24,248 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:29:24,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-08-10T06:29:24,352 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1691648964351_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } 2023-08-10T06:29:24,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691648964352 (current time:1691648964352). 2023-08-10T06:29:24,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:29:24,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1691648964351_default_test-1691648868734 VERSION not specified, setting to 2 2023-08-10T06:29:24,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42d647, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:29:24,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:29:24,362 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:40764, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:29:24,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:29:24,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:29:24,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=81, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1691648964351_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } 2023-08-10T06:29:24,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1691648964351_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 }, snapshot procedure id = 81 2023-08-10T06:29:24,365 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1691648964351_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:29:24,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=81 2023-08-10T06:29:24,366 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1691648964351_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:29:24,368 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1691648964351_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:29:24,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=81 2023-08-10T06:29:24,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=81 2023-08-10T06:29:24,773 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1691648964351_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:29:24,774 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE; SnapshotRegionProcedure acd4d10145cd83e849c8f35f15dd289e}] 2023-08-10T06:29:24,775 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=82, ppid=81, state=RUNNABLE; SnapshotRegionProcedure acd4d10145cd83e849c8f35f15dd289e 2023-08-10T06:29:24,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45101 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=82 2023-08-10T06:29:24,927 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1691648868734,,1691648917151.acd4d10145cd83e849c8f35f15dd289e. 2023-08-10T06:29:24,927 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] regionserver.HRegion(2520): Flush status journal for acd4d10145cd83e849c8f35f15dd289e: 2023-08-10T06:29:24,927 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1691648868734,,1691648917151.acd4d10145cd83e849c8f35f15dd289e. for snapshot_1691648964351_default_test-1691648868734 completed. 2023-08-10T06:29:24,927 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] snapshot.SnapshotManifest(240): Storing 'test-1691648868734,,1691648917151.acd4d10145cd83e849c8f35f15dd289e.' region-info for snapshot=snapshot_1691648964351_default_test-1691648868734 2023-08-10T06:29:24,927 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:29:24,927 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/acd4d10145cd83e849c8f35f15dd289e/f/eb3ec537e8f043cfaae1688b3b8e2162_SeqId_4_] hfiles 2023-08-10T06:29:24,927 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/acd4d10145cd83e849c8f35f15dd289e/f/eb3ec537e8f043cfaae1688b3b8e2162_SeqId_4_ for snapshot=snapshot_1691648964351_default_test-1691648868734 2023-08-10T06:29:24,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=81 2023-08-10T06:29:25,333 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1691648868734,,1691648917151.acd4d10145cd83e849c8f35f15dd289e. 2023-08-10T06:29:25,333 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=82 2023-08-10T06:29:25,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster(3994): Remote procedure done, pid=82 2023-08-10T06:29:25,333 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1691648964351_default_test-1691648868734 on region acd4d10145cd83e849c8f35f15dd289e 2023-08-10T06:29:25,335 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=82, ppid=81, state=RUNNABLE; SnapshotRegionProcedure acd4d10145cd83e849c8f35f15dd289e 2023-08-10T06:29:25,336 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=82, resume processing ppid=81 2023-08-10T06:29:25,337 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=82, ppid=81, state=SUCCESS; SnapshotRegionProcedure acd4d10145cd83e849c8f35f15dd289e in 562 msec 2023-08-10T06:29:25,337 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1691648964351_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:29:25,337 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1691648964351_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:29:25,338 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1691648964351_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:29:25,338 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1691648964351_default_test-1691648868734 2023-08-10T06:29:25,338 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_1691648964351_default_test-1691648868734 2023-08-10T06:29:25,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=81 2023-08-10T06:29:25,747 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1691648964351_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:29:25,752 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1691648964351_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:29:25,753 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_1691648964351_default_test-1691648868734 to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691648964351_default_test-1691648868734 2023-08-10T06:29:26,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=81 2023-08-10T06:29:26,569 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1691648964351_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:29:26,569 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1691648964351_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 }, snapshot procedure id = 81 2023-08-10T06:29:26,570 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=81, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1691648964351_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } in 2.2060 sec 2023-08-10T06:29:26,762 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_table1 2023-08-10T06:29:28,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=81 2023-08-10T06:29:28,472 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1691648960009 2023-08-10T06:29:28,472 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-08-10T06:29:28,472 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-08-10T06:29:28,472 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1691648960009_test-1691648868734 2023-08-10T06:29:28,472 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1691648964351_default_test-1691648868734 to hdfs://localhost:45379/backupUT/backup_1691648960009/default/test-1691648868734/ 2023-08-10T06:29:28,472 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-08-10T06:29:28,495 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:45379, inputRoot=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e 2023-08-10T06:29:28,495 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,33705,1691648856595_-255836071_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:45379/backupUT/backup_1691648960009/default/test-1691648868734, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:45379/backupUT/backup_1691648960009/default/test-1691648868734/.hbase-snapshot/.tmp/snapshot_1691648964351_default_test-1691648868734 2023-08-10T06:29:28,495 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:45379, inputRoot=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e, snapshotDir=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691648964351_default_test-1691648868734. 2023-08-10T06:29:28,502 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691648964351_default_test-1691648868734 to hdfs://localhost:45379/backupUT/backup_1691648960009/default/test-1691648868734/.hbase-snapshot/.tmp/snapshot_1691648964351_default_test-1691648868734 2023-08-10T06:29:28,914 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:28,915 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:28,915 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:28,916 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:29,871 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/hadoop-3025744813419752259.jar 2023-08-10T06:29:29,872 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:29,872 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:29,872 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:29,873 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:29,873 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:29,873 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:29,874 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:29,874 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:29,874 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:29,875 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:29:29,876 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-08-10T06:29:29,876 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-08-10T06:29:29,876 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-08-10T06:29:29,877 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-08-10T06:29:29,877 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-08-10T06:29:29,877 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-08-10T06:29:29,878 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-08-10T06:29:29,878 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-08-10T06:29:29,879 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-08-10T06:29:29,879 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-08-10T06:29:29,879 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-08-10T06:29:29,880 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:29:29,880 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:29:29,881 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:29:29,881 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:29:29,881 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:29:29,882 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:29:29,883 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:29:29,925 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1691648964351_default_test-1691648868734' hfile list 2023-08-10T06:29:29,927 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-08-10T06:29:29,995 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691648969962/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-08-10T06:29:29,995 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691648969962/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-08-10T06:29:30,046 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-08-10T06:29:30,070 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/acd4d10145cd83e849c8f35f15dd289e/f/eb3ec537e8f043cfaae1688b3b8e2162_SeqId_4_ output=hdfs://localhost:45379/backupUT/backup_1691648960009/default/test-1691648868734/archive/data/default/test-1691648868734/acd4d10145cd83e849c8f35f15dd289e/f/eb3ec537e8f043cfaae1688b3b8e2162_SeqId_4_ 2023-08-10T06:29:30,070 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8724 (8.5 K) time=0sec 8.320M/sec 2023-08-10T06:29:31,030 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-08-10T06:29:31,031 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-08-10T06:29:31,039 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1691648964351_default_test-1691648868734 2023-08-10T06:29:31,039 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1691648964351_default_test-1691648868734 finished. 2023-08-10T06:29:31,039 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(1014): test-1691648868734 2023-08-10T06:29:31,047 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691648960009 2023-08-10T06:29:31,047 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-08-10T06:29:31,453 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:45379/backupUT/backup_1691648960009/.backup.manifest 2023-08-10T06:29:31,453 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(358): Backup backup_1691648960009 finished: type=FULL,tablelist=test-1691648868734,targetRootDir=hdfs://localhost:45379/backupUT,startts=1691648964157,completets=1691648971047,bytescopied=0 2023-08-10T06:29:31,453 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-08-10T06:29:31,453 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1691648964351_default_test-1691648868734 2023-08-10T06:29:31,454 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1691648964351_default_test-1691648868734" 2023-08-10T06:29:31,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1691648964351_default_test-1691648868734 2023-08-10T06:29:31,457 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1691648964351_default_test-1691648868734 for backup backup_1691648960009 succeeded. 2023-08-10T06:29:31,457 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-08-10T06:29:31,461 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-08-10T06:29:31,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-08-10T06:29:31,464 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-08-10T06:29:31,465 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-08-10T06:29:31,471 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(376): Backup backup_1691648960009 completed. 2023-08-10T06:29:31,471 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:29:31,483 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] backup.TestFullRestore(196): backup complete 2023-08-10T06:29:31,484 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] backup.RestoreDriver(101): Found -overwrite option in restore command, will overwrite to existing table if any in the restore target 2023-08-10T06:29:31,492 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bd71863, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:29:31,493 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:29:31,495 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:53236, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:29:31,497 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:45379/backupUT/backup_1691648960009/.backup.manifest 2023-08-10T06:29:31,499 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691648960009/.backup.manifest 2023-08-10T06:29:31,500 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:45379/backupUT/backup_1691648960009/.backup.manifest 2023-08-10T06:29:31,502 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691648960009/.backup.manifest 2023-08-10T06:29:31,502 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(148): Restoring 'test-1691648868734' to 'test-1691648868734' from full backup image hdfs://localhost:45379/backupUT/backup_1691648960009/default/test-1691648868734 2023-08-10T06:29:31,508 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] util.RestoreTool(476): Truncating exising target table 'test-1691648868734', preserving region splits 2023-08-10T06:29:31,509 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:29:31,510 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:60042, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:29:31,511 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster$12(2663): Client=jenkins//172.31.12.81 disable test-1691648868734 2023-08-10T06:29:31,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=83, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=test-1691648868734 2023-08-10T06:29:31,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=83 2023-08-10T06:29:31,515 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1691648868734","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648971515"}]},"ts":"1691648971515"} 2023-08-10T06:29:31,516 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1691648868734, state=DISABLING in hbase:meta 2023-08-10T06:29:31,517 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(293): Set test-1691648868734 to state=DISABLING 2023-08-10T06:29:31,518 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=test-1691648868734, region=acd4d10145cd83e849c8f35f15dd289e, UNASSIGN}] 2023-08-10T06:29:31,519 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=test-1691648868734, region=acd4d10145cd83e849c8f35f15dd289e, UNASSIGN 2023-08-10T06:29:31,520 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=84 updating hbase:meta row=acd4d10145cd83e849c8f35f15dd289e, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:31,521 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE; CloseRegionProcedure acd4d10145cd83e849c8f35f15dd289e, server=jenkins-hbase3.apache.org,45101,1691648859755}] 2023-08-10T06:29:31,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=83 2023-08-10T06:29:31,671 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(119): Close acd4d10145cd83e849c8f35f15dd289e 2023-08-10T06:29:31,671 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1668): Closing acd4d10145cd83e849c8f35f15dd289e, disabling compactions & flushes 2023-08-10T06:29:31,671 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1690): Closing region test-1691648868734,,1691648917151.acd4d10145cd83e849c8f35f15dd289e. 2023-08-10T06:29:31,671 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1691648868734,,1691648917151.acd4d10145cd83e849c8f35f15dd289e. 2023-08-10T06:29:31,672 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1778): Acquired close lock on test-1691648868734,,1691648917151.acd4d10145cd83e849c8f35f15dd289e. after waiting 0 ms 2023-08-10T06:29:31,672 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1788): Updates disabled for region test-1691648868734,,1691648917151.acd4d10145cd83e849c8f35f15dd289e. 2023-08-10T06:29:31,673 DEBUG [StoreFileCloser-acd4d10145cd83e849c8f35f15dd289e-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: eb3ec537e8f043cfaae1688b3b8e2162_SeqId_4_ 2023-08-10T06:29:31,678 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/acd4d10145cd83e849c8f35f15dd289e/recovered.edits/10.seqid, newMaxSeqId=10, maxSeqId=1 2023-08-10T06:29:31,679 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:29:31,679 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1909): Closed test-1691648868734,,1691648917151.acd4d10145cd83e849c8f35f15dd289e. 2023-08-10T06:29:31,679 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1622): Region close journal for acd4d10145cd83e849c8f35f15dd289e: 2023-08-10T06:29:31,681 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(163): Closed acd4d10145cd83e849c8f35f15dd289e 2023-08-10T06:29:31,681 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=84 updating hbase:meta row=acd4d10145cd83e849c8f35f15dd289e, regionState=CLOSED 2023-08-10T06:29:31,684 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=85, resume processing ppid=84 2023-08-10T06:29:31,684 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=85, ppid=84, state=SUCCESS; CloseRegionProcedure acd4d10145cd83e849c8f35f15dd289e, server=jenkins-hbase3.apache.org,45101,1691648859755 in 162 msec 2023-08-10T06:29:31,685 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=84, resume processing ppid=83 2023-08-10T06:29:31,685 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=84, ppid=83, state=SUCCESS; TransitRegionStateProcedure table=test-1691648868734, region=acd4d10145cd83e849c8f35f15dd289e, UNASSIGN in 166 msec 2023-08-10T06:29:31,685 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1691648868734","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648971685"}]},"ts":"1691648971685"} 2023-08-10T06:29:31,686 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1691648868734, state=DISABLED in hbase:meta 2023-08-10T06:29:31,688 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(305): Set test-1691648868734 to state=DISABLED 2023-08-10T06:29:31,689 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=83, state=SUCCESS; DisableTableProcedure table=test-1691648868734 in 177 msec 2023-08-10T06:29:31,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=83 2023-08-10T06:29:31,817 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster$6(2479): Client=jenkins//172.31.12.81 truncate test-1691648868734 2023-08-10T06:29:31,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=86, state=RUNNABLE:TRUNCATE_TABLE_PRE_OPERATION; TruncateTableProcedure (table=test-1691648868734 preserveSplits=true) 2023-08-10T06:29:31,820 DEBUG [PEWorker-5 {}] procedure.TruncateTableProcedure(87): waiting for 'test-1691648868734' regions in transition 2023-08-10T06:29:31,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=86 2023-08-10T06:29:31,823 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/acd4d10145cd83e849c8f35f15dd289e 2023-08-10T06:29:31,824 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/acd4d10145cd83e849c8f35f15dd289e/f, FileablePath, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/acd4d10145cd83e849c8f35f15dd289e/recovered.edits] 2023-08-10T06:29:31,829 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/acd4d10145cd83e849c8f35f15dd289e/f/eb3ec537e8f043cfaae1688b3b8e2162_SeqId_4_ to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/archive/data/default/test-1691648868734/acd4d10145cd83e849c8f35f15dd289e/f/eb3ec537e8f043cfaae1688b3b8e2162_SeqId_4_ 2023-08-10T06:29:31,832 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/acd4d10145cd83e849c8f35f15dd289e/recovered.edits/10.seqid to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/archive/data/default/test-1691648868734/acd4d10145cd83e849c8f35f15dd289e/recovered.edits/10.seqid 2023-08-10T06:29:31,832 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/acd4d10145cd83e849c8f35f15dd289e 2023-08-10T06:29:31,833 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(293): Archived test-1691648868734 regions 2023-08-10T06:29:31,837 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of test-1691648868734 from hbase:meta 2023-08-10T06:29:31,839 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(387): Removing 'test-1691648868734' descriptor. 2023-08-10T06:29:31,839 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(377): Removing 'test-1691648868734' from region states. 2023-08-10T06:29:31,839 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"test-1691648868734,,1691648917151.acd4d10145cd83e849c8f35f15dd289e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1691648971839"}]},"ts":"9223372036854775807"} 2023-08-10T06:29:31,841 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-08-10T06:29:31,841 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => acd4d10145cd83e849c8f35f15dd289e, NAME => 'test-1691648868734,,1691648917151.acd4d10145cd83e849c8f35f15dd289e.', STARTKEY => '', ENDKEY => ''}] 2023-08-10T06:29:31,841 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(381): Marking 'test-1691648868734' as deleted. 2023-08-10T06:29:31,841 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"test-1691648868734","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1691648971841"}]},"ts":"9223372036854775807"} 2023-08-10T06:29:31,843 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1678): Deleted table test-1691648868734 state from META 2023-08-10T06:29:31,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=86 2023-08-10T06:29:32,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=86 2023-08-10T06:29:32,251 INFO [RegionOpenAndInit-test-1691648868734-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 513ad41c456db08f6287ccd38ca6b19c, NAME => 'test-1691648868734,,1691648971834.513ad41c456db08f6287ccd38ca6b19c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1691648868734', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e 2023-08-10T06:29:32,316 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-08-10T06:29:32,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=86 2023-08-10T06:29:32,657 DEBUG [RegionOpenAndInit-test-1691648868734-pool-0 {}] regionserver.HRegion(888): Instantiated test-1691648868734,,1691648971834.513ad41c456db08f6287ccd38ca6b19c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:29:32,657 DEBUG [RegionOpenAndInit-test-1691648868734-pool-0 {}] regionserver.HRegion(1668): Closing 513ad41c456db08f6287ccd38ca6b19c, disabling compactions & flushes 2023-08-10T06:29:32,657 INFO [RegionOpenAndInit-test-1691648868734-pool-0 {}] regionserver.HRegion(1690): Closing region test-1691648868734,,1691648971834.513ad41c456db08f6287ccd38ca6b19c. 2023-08-10T06:29:32,657 DEBUG [RegionOpenAndInit-test-1691648868734-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1691648868734,,1691648971834.513ad41c456db08f6287ccd38ca6b19c. 2023-08-10T06:29:32,657 DEBUG [RegionOpenAndInit-test-1691648868734-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on test-1691648868734,,1691648971834.513ad41c456db08f6287ccd38ca6b19c. after waiting 0 ms 2023-08-10T06:29:32,658 DEBUG [RegionOpenAndInit-test-1691648868734-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region test-1691648868734,,1691648971834.513ad41c456db08f6287ccd38ca6b19c. 2023-08-10T06:29:32,658 INFO [RegionOpenAndInit-test-1691648868734-pool-0 {}] regionserver.HRegion(1909): Closed test-1691648868734,,1691648971834.513ad41c456db08f6287ccd38ca6b19c. 2023-08-10T06:29:32,658 DEBUG [RegionOpenAndInit-test-1691648868734-pool-0 {}] regionserver.HRegion(1622): Region close journal for 513ad41c456db08f6287ccd38ca6b19c: 2023-08-10T06:29:32,659 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"test-1691648868734,,1691648971834.513ad41c456db08f6287ccd38ca6b19c.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1691648972659"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691648972659"}]},"ts":"1691648972659"} 2023-08-10T06:29:32,660 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:29:32,661 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1691648868734","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648972661"}]},"ts":"1691648972661"} 2023-08-10T06:29:32,662 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1691648868734, state=ENABLING in hbase:meta 2023-08-10T06:29:32,667 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1691648868734, region=513ad41c456db08f6287ccd38ca6b19c, ASSIGN}] 2023-08-10T06:29:32,668 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=87, ppid=86, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1691648868734, region=513ad41c456db08f6287ccd38ca6b19c, ASSIGN 2023-08-10T06:29:32,669 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=87, ppid=86, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=test-1691648868734, region=513ad41c456db08f6287ccd38ca6b19c, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45101,1691648859755; forceNewPlan=false, retain=false 2023-08-10T06:29:32,820 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=87 updating hbase:meta row=513ad41c456db08f6287ccd38ca6b19c, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:32,822 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE; OpenRegionProcedure 513ad41c456db08f6287ccd38ca6b19c, server=jenkins-hbase3.apache.org,45101,1691648859755}] 2023-08-10T06:29:32,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=86 2023-08-10T06:29:32,977 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(130): Open test-1691648868734,,1691648971834.513ad41c456db08f6287ccd38ca6b19c. 2023-08-10T06:29:32,977 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7254): Opening region: {ENCODED => 513ad41c456db08f6287ccd38ca6b19c, NAME => 'test-1691648868734,,1691648971834.513ad41c456db08f6287ccd38ca6b19c.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:29:32,977 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:29:32,978 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1691648868734 513ad41c456db08f6287ccd38ca6b19c 2023-08-10T06:29:32,978 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(888): Instantiated test-1691648868734,,1691648971834.513ad41c456db08f6287ccd38ca6b19c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:29:32,978 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7296): checking encryption for 513ad41c456db08f6287ccd38ca6b19c 2023-08-10T06:29:32,978 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7299): checking classloading for 513ad41c456db08f6287ccd38ca6b19c 2023-08-10T06:29:32,979 INFO [StoreOpener-513ad41c456db08f6287ccd38ca6b19c-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 513ad41c456db08f6287ccd38ca6b19c 2023-08-10T06:29:32,981 INFO [StoreOpener-513ad41c456db08f6287ccd38ca6b19c-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 513ad41c456db08f6287ccd38ca6b19c columnFamilyName f 2023-08-10T06:29:32,981 DEBUG [StoreOpener-513ad41c456db08f6287ccd38ca6b19c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:29:32,981 INFO [StoreOpener-513ad41c456db08f6287ccd38ca6b19c-1 {}] regionserver.HStore(324): Store=513ad41c456db08f6287ccd38ca6b19c/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:29:32,982 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/513ad41c456db08f6287ccd38ca6b19c 2023-08-10T06:29:32,982 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/513ad41c456db08f6287ccd38ca6b19c 2023-08-10T06:29:32,984 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1076): writing seq id for 513ad41c456db08f6287ccd38ca6b19c 2023-08-10T06:29:32,986 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/513ad41c456db08f6287ccd38ca6b19c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:29:32,986 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1093): Opened 513ad41c456db08f6287ccd38ca6b19c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11384492160, jitterRate=0.06026345491409302}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-08-10T06:29:32,987 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(992): Region open journal for 513ad41c456db08f6287ccd38ca6b19c: 2023-08-10T06:29:32,987 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2552): Post open deploy tasks for test-1691648868734,,1691648971834.513ad41c456db08f6287ccd38ca6b19c., pid=88, masterSystemTime=1691648972973 2023-08-10T06:29:32,989 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2579): Finished post open deploy task for test-1691648868734,,1691648971834.513ad41c456db08f6287ccd38ca6b19c. 2023-08-10T06:29:32,989 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(158): Opened test-1691648868734,,1691648971834.513ad41c456db08f6287ccd38ca6b19c. 2023-08-10T06:29:32,989 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=87 updating hbase:meta row=513ad41c456db08f6287ccd38ca6b19c, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:32,992 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=88, resume processing ppid=87 2023-08-10T06:29:32,992 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=88, ppid=87, state=SUCCESS; OpenRegionProcedure 513ad41c456db08f6287ccd38ca6b19c, server=jenkins-hbase3.apache.org,45101,1691648859755 in 169 msec 2023-08-10T06:29:32,993 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=87, resume processing ppid=86 2023-08-10T06:29:32,993 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=87, ppid=86, state=SUCCESS; TransitRegionStateProcedure table=test-1691648868734, region=513ad41c456db08f6287ccd38ca6b19c, ASSIGN in 325 msec 2023-08-10T06:29:32,993 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1691648868734","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648972993"}]},"ts":"1691648972993"} 2023-08-10T06:29:32,994 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1691648868734, state=ENABLED in hbase:meta 2023-08-10T06:29:32,997 DEBUG [PEWorker-1 {}] procedure.TruncateTableProcedure(145): truncate 'test-1691648868734' completed 2023-08-10T06:29:32,998 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=86, state=SUCCESS; TruncateTableProcedure (table=test-1691648868734 preserveSplits=true) in 1.1800 sec 2023-08-10T06:29:33,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=86 2023-08-10T06:29:33,931 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:45379/backupUT/backup_1691648960009/default/test-1691648868734/archive/data/default/test-1691648868734/acd4d10145cd83e849c8f35f15dd289e from hbase tables test-1691648868734 to tables test-1691648868734 2023-08-10T06:29:33,931 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceRestoreJob(72): Restore test-1691648868734 into test-1691648868734 2023-08-10T06:29:33,933 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-default-test-1691648868734-1691648973931 from hdfs://localhost:45379/backupUT/backup_1691648960009/default/test-1691648868734/archive/data/default/test-1691648868734/acd4d10145cd83e849c8f35f15dd289e 2023-08-10T06:29:33,937 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23c3832a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:29:33,938 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:29:33,939 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:60058, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:29:33,940 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-08-10T06:29:33,940 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table test-1691648868734 2023-08-10T06:29:33,943 DEBUG [hconnection-0x68b6efd7-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:29:33,944 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:53246, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:29:33,946 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for test-1691648868734: 2023-08-10T06:29:33,946 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-08-10T06:29:33,946 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/partitions_39a19274-8e69-4c41-b0e6-fb856f7a3801 2023-08-10T06:29:34,353 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:34,353 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:34,354 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:34,354 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:35,320 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/hadoop-7471864515932636604.jar 2023-08-10T06:29:35,321 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:35,321 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:35,322 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:35,322 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:35,322 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:35,323 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:35,323 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:35,324 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:35,324 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:35,325 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:29:35,325 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-08-10T06:29:35,326 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-08-10T06:29:35,326 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-08-10T06:29:35,326 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-08-10T06:29:35,327 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-08-10T06:29:35,327 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-08-10T06:29:35,327 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-08-10T06:29:35,328 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-08-10T06:29:35,328 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-08-10T06:29:35,329 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-08-10T06:29:35,329 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-08-10T06:29:35,330 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:35,330 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:35,330 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:35,330 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:35,331 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:35,331 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:35,332 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:29:35,332 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: test-1691648868734 2023-08-10T06:29:35,332 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:29:35,332 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-08-10T06:29:35,332 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-08-10T06:29:35,333 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:29:35,344 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-08-10T06:29:35,439 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691648975404/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-08-10T06:29:35,439 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691648975404/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-08-10T06:29:35,487 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:45379/backupUT/backup_1691648960009/default/test-1691648868734/archive/data/default/test-1691648868734/acd4d10145cd83e849c8f35f15dd289e/f/eb3ec537e8f043cfaae1688b3b8e2162_SeqId_4_ 2023-08-10T06:29:35,491 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-08-10T06:29:35,493 DEBUG [LocalJobRunner Map Task Executor #0 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: eb3ec537e8f043cfaae1688b3b8e2162_SeqId_4_ 2023-08-10T06:29:35,510 DEBUG [pool-329-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46a95ba3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:29:35,512 DEBUG [hconnection-0x21d56aa-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:29:35,514 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:53260, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:29:35,517 DEBUG [pool-329-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:29:35,517 DEBUG [pool-329-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-08-10T06:29:35,517 DEBUG [pool-329-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-08-10T06:29:35,523 DEBUG [pool-329-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-default-test-1691648868734-1691648973931/_temporary/0/_temporary/attempt_local426465149_0011_r_000000_0/f/3c51979652ab4bbbb2a342b0ecf5ea0a is 37, key is row10/f:q1/1691648871386/Put/seqid=0 2023-08-10T06:29:35,524 WARN [Thread-1502 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:45101] Specified favored nodes: [jenkins-hbase3.apache.org:45101] 2023-08-10T06:29:36,486 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-default-test-1691648868734-1691648973931 2023-08-10T06:29:36,491 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9a2d347, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:29:36,492 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:29:36,493 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:53272, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:29:36,498 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-default-test-1691648868734-1691648973931/_SUCCESS 2023-08-10T06:29:36,502 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:29:36,503 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:60066, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:29:36,512 DEBUG [LoadIncrementalHFiles-0 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 3c51979652ab4bbbb2a342b0ecf5ea0a 2023-08-10T06:29:36,512 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-default-test-1691648868734-1691648973931/f/3c51979652ab4bbbb2a342b0ecf5ea0a first=Optional[row0] last=Optional[row98] 2023-08-10T06:29:36,513 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=test-1691648868734,,1691648971834.513ad41c456db08f6287ccd38ca6b19c., hostname=jenkins-hbase3.apache.org,45101,1691648859755, seqNum=2 for row with hfile group [{f,hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-default-test-1691648868734-1691648973931/f/3c51979652ab4bbbb2a342b0ecf5ea0a}] 2023-08-10T06:29:36,520 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-default-test-1691648868734-1691648973931/f/3c51979652ab4bbbb2a342b0ecf5ea0a for inclusion in 513ad41c456db08f6287ccd38ca6b19c/f 2023-08-10T06:29:36,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-08-10T06:29:36,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HStore(625): Region bounds: first= last= 2023-08-10T06:29:36,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 3c51979652ab4bbbb2a342b0ecf5ea0a 2023-08-10T06:29:36,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HRegion(2520): Flush status journal for 513ad41c456db08f6287ccd38ca6b19c: 2023-08-10T06:29:36,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-default-test-1691648868734-1691648973931/f/3c51979652ab4bbbb2a342b0ecf5ea0a to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/staging/jenkins__test-1691648868734__kseiovt9b1kgsufmqiod9568tsc8h7hc2govj57054h8uvaqoicke3rqepffbd2u/f/3c51979652ab4bbbb2a342b0ecf5ea0a 2023-08-10T06:29:36,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/staging/jenkins__test-1691648868734__kseiovt9b1kgsufmqiod9568tsc8h7hc2govj57054h8uvaqoicke3rqepffbd2u/f/3c51979652ab4bbbb2a342b0ecf5ea0a as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/513ad41c456db08f6287ccd38ca6b19c/f/8eec1fbc8346444d9f3bf9d53b64b2b2_SeqId_4_ 2023-08-10T06:29:36,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@314c12b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:29:36,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:29:36,533 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:60078, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:29:36,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:29:36,537 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:53278, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:29:36,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] impl.BackupSystemTable(408): write bulk load descriptor to backup test-1691648868734 with 1 entries 2023-08-10T06:29:36,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] impl.BackupSystemTable(1582): writing raw bulk path hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/513ad41c456db08f6287ccd38ca6b19c/f/8eec1fbc8346444d9f3bf9d53b64b2b2_SeqId_4_ for test-1691648868734 513ad41c456db08f6287ccd38ca6b19c 2023-08-10T06:29:36,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] impl.BackupSystemTable(415): written 1 rows for bulk load of test-1691648868734 2023-08-10T06:29:36,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:29:36,553 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/staging/jenkins__test-1691648868734__kseiovt9b1kgsufmqiod9568tsc8h7hc2govj57054h8uvaqoicke3rqepffbd2u/f/3c51979652ab4bbbb2a342b0ecf5ea0a into 513ad41c456db08f6287ccd38ca6b19c/f as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/513ad41c456db08f6287ccd38ca6b19c/f/8eec1fbc8346444d9f3bf9d53b64b2b2_SeqId_4_ - updating store file list. 2023-08-10T06:29:36,557 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/513ad41c456db08f6287ccd38ca6b19c/f/8eec1fbc8346444d9f3bf9d53b64b2b2_SeqId_4_ into 513ad41c456db08f6287ccd38ca6b19c/f 2023-08-10T06:29:36,557 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/staging/jenkins__test-1691648868734__kseiovt9b1kgsufmqiod9568tsc8h7hc2govj57054h8uvaqoicke3rqepffbd2u/f/3c51979652ab4bbbb2a342b0ecf5ea0a into 513ad41c456db08f6287ccd38ca6b19c/f (new location: hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/513ad41c456db08f6287ccd38ca6b19c/f/8eec1fbc8346444d9f3bf9d53b64b2b2_SeqId_4_) 2023-08-10T06:29:36,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/staging/jenkins__test-1691648868734__kseiovt9b1kgsufmqiod9568tsc8h7hc2govj57054h8uvaqoicke3rqepffbd2u/f/3c51979652ab4bbbb2a342b0ecf5ea0a 2023-08-10T06:29:36,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2986d202, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:29:36,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:29:36,566 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:60084, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-08-10T06:29:36,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:29:36,571 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:53284, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-08-10T06:29:36,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] impl.BackupSystemTable(388): write bulk load descriptor to backup test-1691648868734 with 1 entries 2023-08-10T06:29:36,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] impl.BackupSystemTable(1503): writing done bulk path hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/513ad41c456db08f6287ccd38ca6b19c/f/8eec1fbc8346444d9f3bf9d53b64b2b2_SeqId_4_ for test-1691648868734 513ad41c456db08f6287ccd38ca6b19c 2023-08-10T06:29:36,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] impl.BackupSystemTable(394): written 1 rows for bulk load of test-1691648868734 2023-08-10T06:29:36,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:29:36,586 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:29:36,586 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-08-10T06:29:36,586 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-08-10T06:29:36,587 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(233): Backup: backup_1691648960009 hdfs://localhost:45379/backupUT/backup_1691648960009/default/test-1691648868734/ 2023-08-10T06:29:36,587 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-08-10T06:29:36,587 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:29:36,608 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreSingleOverwriteCommand Thread=604 (was 594) - Thread LEAK? -, OpenFileDescriptor=688 (was 670) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=119 (was 126), ProcessCount=169 (was 170), AvailableMemoryMB=1475 (was 1583) 2023-08-10T06:29:36,608 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(130): Thread=604 is superior to 500 2023-08-10T06:29:36,623 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreMultipleDNECommand Thread=604, OpenFileDescriptor=688, MaxFileDescriptor=60000, SystemLoadAverage=119, ProcessCount=169, AvailableMemoryMB=1474 2023-08-10T06:29:36,623 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(130): Thread=604 is superior to 500 2023-08-10T06:29:36,623 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] backup.TestFullRestore(322): test restore fails on multiple tables that do not exist: command-line 2023-08-10T06:29:36,628 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57c31b96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:29:36,632 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:29:36,633 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:53298, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:29:36,635 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:29:36,636 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:60100, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:29:36,648 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-08-10T06:29:36,653 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:29:36,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691648976653 (current time:1691648976653). 2023-08-10T06:29:36,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:29:36,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-08-10T06:29:36,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c3ed061, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:29:36,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:29:36,660 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:53306, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:29:36,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:29:36,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:29:36,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=89, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:29:36,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 89 2023-08-10T06:29:36,664 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:29:36,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=89 2023-08-10T06:29:36,665 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:29:36,666 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:29:36,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=89 2023-08-10T06:29:36,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=89 2023-08-10T06:29:37,074 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:29:37,074 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72}] 2023-08-10T06:29:37,075 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=90, ppid=89, state=RUNNABLE; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:29:37,181 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-08-10T06:29:37,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45101 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=90 2023-08-10T06:29:37,227 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. 2023-08-10T06:29:37,227 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.HRegion(2819): Flushing b9c493f2c9fbef45de19e8268a534e72 2/2 column families, dataSize=1.14 KB heapSize=2.22 KB 2023-08-10T06:29:37,232 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/meta/24713cefebe0499c812c6d209dea0fd6 is 169, key is trslm:hdfs://localhost:45379/backupUT\x00test-1691648868734/meta:log-roll-map/1691648971042/Put/seqid=0 2023-08-10T06:29:37,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=89 2023-08-10T06:29:37,642 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=451 B at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/meta/24713cefebe0499c812c6d209dea0fd6 2023-08-10T06:29:37,647 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 24713cefebe0499c812c6d209dea0fd6 2023-08-10T06:29:37,651 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/session/68fd0cf6c3094cc7af1a5176829fd9a4 is 452, key is session:backup_1691648960009/session:context/1691648971464/Put/seqid=0 2023-08-10T06:29:37,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=89 2023-08-10T06:29:38,055 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=721 B at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/session/68fd0cf6c3094cc7af1a5176829fd9a4 2023-08-10T06:29:38,060 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 68fd0cf6c3094cc7af1a5176829fd9a4 2023-08-10T06:29:38,061 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/meta/24713cefebe0499c812c6d209dea0fd6 as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/24713cefebe0499c812c6d209dea0fd6 2023-08-10T06:29:38,066 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/24713cefebe0499c812c6d209dea0fd6, entries=4, sequenceid=72, filesize=5.5 K 2023-08-10T06:29:38,067 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/session/68fd0cf6c3094cc7af1a5176829fd9a4 as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/68fd0cf6c3094cc7af1a5176829fd9a4 2023-08-10T06:29:38,072 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/68fd0cf6c3094cc7af1a5176829fd9a4, entries=2, sequenceid=72, filesize=5.4 K 2023-08-10T06:29:38,073 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1172, heapSize ~2.19 KB/2240, currentSize=0 B/0 for b9c493f2c9fbef45de19e8268a534e72 in 845ms, sequenceid=72, compaction requested=true 2023-08-10T06:29:38,073 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.HRegion(2520): Flush status journal for b9c493f2c9fbef45de19e8268a534e72: 2023-08-10T06:29:38,073 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. for snapshot_backup_system completed. 2023-08-10T06:29:38,073 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72.' region-info for snapshot=snapshot_backup_system 2023-08-10T06:29:38,073 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:29:38,073 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/bc10682e7f36491d941e2fe62e2e4100, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/9d9a4feb1c4e44d99cc63e3f68878731, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/f04a99990b2c429eaf1096ef413be3b8, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/9e97dc38b66b476f952990b23fa0a3ca, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/930c578fe3f543e1b6308abfb31c7246, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/24713cefebe0499c812c6d209dea0fd6] hfiles 2023-08-10T06:29:38,073 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (1/6): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/bc10682e7f36491d941e2fe62e2e4100 for snapshot=snapshot_backup_system 2023-08-10T06:29:38,073 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (2/6): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/9d9a4feb1c4e44d99cc63e3f68878731 for snapshot=snapshot_backup_system 2023-08-10T06:29:38,073 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (3/6): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/f04a99990b2c429eaf1096ef413be3b8 for snapshot=snapshot_backup_system 2023-08-10T06:29:38,073 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (4/6): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/9e97dc38b66b476f952990b23fa0a3ca for snapshot=snapshot_backup_system 2023-08-10T06:29:38,073 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (5/6): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/930c578fe3f543e1b6308abfb31c7246 for snapshot=snapshot_backup_system 2023-08-10T06:29:38,073 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (6/6): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/24713cefebe0499c812c6d209dea0fd6 for snapshot=snapshot_backup_system 2023-08-10T06:29:38,073 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/e88678d5b4b14eecabac817daf12539e, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/37561b9a8ae2406bb6d053c33aeff31c, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/883911c19cab473f8bf801f934c90bc1, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/0c7705a45c554d6a97fa37fac319ceff, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/398a15bfffb84acfadb1be252c2e133a, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/adceb6320b02451abd02beed6c69c000, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/68fd0cf6c3094cc7af1a5176829fd9a4] hfiles 2023-08-10T06:29:38,073 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (1/7): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/e88678d5b4b14eecabac817daf12539e for snapshot=snapshot_backup_system 2023-08-10T06:29:38,073 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (2/7): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/37561b9a8ae2406bb6d053c33aeff31c for snapshot=snapshot_backup_system 2023-08-10T06:29:38,073 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (3/7): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/883911c19cab473f8bf801f934c90bc1 for snapshot=snapshot_backup_system 2023-08-10T06:29:38,073 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (4/7): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/0c7705a45c554d6a97fa37fac319ceff for snapshot=snapshot_backup_system 2023-08-10T06:29:38,073 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (5/7): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/398a15bfffb84acfadb1be252c2e133a for snapshot=snapshot_backup_system 2023-08-10T06:29:38,073 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (6/7): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/adceb6320b02451abd02beed6c69c000 for snapshot=snapshot_backup_system 2023-08-10T06:29:38,073 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (7/7): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/68fd0cf6c3094cc7af1a5176829fd9a4 for snapshot=snapshot_backup_system 2023-08-10T06:29:38,478 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. 2023-08-10T06:29:38,478 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=90 2023-08-10T06:29:38,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster(3994): Remote procedure done, pid=90 2023-08-10T06:29:38,479 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:29:38,480 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=90, ppid=89, state=RUNNABLE; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:29:38,482 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=90, resume processing ppid=89 2023-08-10T06:29:38,482 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:29:38,482 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=90, ppid=89, state=SUCCESS; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72 in 1.4070 sec 2023-08-10T06:29:38,483 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:29:38,483 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:29:38,483 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-08-10T06:29:38,484 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_backup_system 2023-08-10T06:29:38,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=89 2023-08-10T06:29:38,896 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:29:38,907 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:29:38,908 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_backup_system 2023-08-10T06:29:39,724 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:29:39,725 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 89 2023-08-10T06:29:39,726 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=89, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 3.0630 sec 2023-08-10T06:29:40,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=89 2023-08-10T06:29:40,770 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(115): Backup backup_1691648976628 started at 1691648980770. 2023-08-10T06:29:40,772 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(119): Backup session backup_1691648976628 has been started. 2023-08-10T06:29:40,777 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-08-10T06:29:40,777 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-08-10T06:29:40,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-08-10T06:29:40,777 INFO [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-08-10T06:29:40,778 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-08-10T06:29:40,778 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-08-10T06:29:40,778 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-08-10T06:29:40,779 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:29:40,780 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:40,780 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:29:40,780 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:29:40,781 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-08-10T06:29:40,781 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-08-10T06:29:40,781 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-08-10T06:29:40,781 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:29:40,781 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-08-10T06:29:40,782 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-08-10T06:29:40,782 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-08-10T06:29:40,782 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-08-10T06:29:40,782 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-08-10T06:29:40,782 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-08-10T06:29:40,782 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-08-10T06:29:40,783 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,45101,1691648859755' joining acquired barrier for procedure (rolllog) in zk 2023-08-10T06:29:40,784 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-08-10T06:29:40,784 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:40,785 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:29:40,785 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:29:40,785 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-08-10T06:29:40,785 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:29:40,785 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:29:40,785 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:29:40,786 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:40,786 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:29:40,786 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,45101,1691648859755' joining acquired barrier for procedure 'rolllog' on coordinator 2023-08-10T06:29:40,786 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@2f37b8a6[Count = 0] remaining members to acquire global barrier 2023-08-10T06:29:40,786 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-08-10T06:29:40,787 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-08-10T06:29:40,788 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-08-10T06:29:40,788 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-08-10T06:29:40,788 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-08-10T06:29:40,788 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-08-10T06:29:40,788 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-08-10T06:29:40,788 DEBUG [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:40,788 INFO [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1691648964205 highest: 1691648964205 on jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:40,788 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta:.meta(num 1691648964181) roll requested 2023-08-10T06:29:40,802 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691648980788.meta, exclude list is [], retry=0 2023-08-10T06:29:40,806 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK] 2023-08-10T06:29:40,808 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691648964181.meta with entries=12, filesize=3.07 KB; new WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691648980788.meta 2023-08-10T06:29:40,808 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK]] 2023-08-10T06:29:40,808 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691648964181.meta is not closed yet, will try archiving it next time 2023-08-10T06:29:40,808 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C45101%2C1691648859755:(num 1691648964205) roll requested 2023-08-10T06:29:40,810 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691648964181.meta not finished, retry = 0 2023-08-10T06:29:40,822 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691648980808, exclude list is [], retry=0 2023-08-10T06:29:40,826 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK] 2023-08-10T06:29:40,827 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691648964205 with entries=15, filesize=4.76 KB; new WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691648980808 2023-08-10T06:29:40,827 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK]] 2023-08-10T06:29:40,827 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691648964205 is not closed yet, will try archiving it next time 2023-08-10T06:29:40,827 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691648947344 to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/oldWALs/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691648947344 2023-08-10T06:29:40,828 DEBUG [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-08-10T06:29:40,828 INFO [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1691648980808 on jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:40,829 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691648964205 not finished, retry = 0 2023-08-10T06:29:40,836 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-08-10T06:29:40,836 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,45101,1691648859755' in zk 2023-08-10T06:29:40,840 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-08-10T06:29:40,840 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:40,840 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:29:40,840 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:29:40,840 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:29:40,840 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-08-10T06:29:40,841 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:29:40,841 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:29:40,841 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:29:40,842 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:40,842 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:29:40,842 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:29:40,842 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:40,843 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,45101,1691648859755': 2023-08-10T06:29:40,843 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,45101,1691648859755' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-08-10T06:29:40,843 INFO [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-08-10T06:29:40,843 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-08-10T06:29:40,843 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-08-10T06:29:40,843 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-08-10T06:29:40,843 INFO [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-08-10T06:29:40,845 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-08-10T06:29:40,845 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-08-10T06:29:40,845 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:29:40,845 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:29:40,845 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:29:40,845 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:29:40,845 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:29:40,845 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:29:40,846 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:29:40,846 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:29:40,846 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:29:40,846 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:29:40,847 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:40,847 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:29:40,847 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:29:40,848 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:40,850 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:29:40,850 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:29:40,851 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:29:40,851 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-08-10T06:29:40,851 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-08-10T06:29:40,851 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:29:40,851 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:29:40,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-08-10T06:29:40,955 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1691648980955_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } 2023-08-10T06:29:40,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691648980955 (current time:1691648980955). 2023-08-10T06:29:40,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:29:40,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1691648980955_ns2_test-16916488687341 VERSION not specified, setting to 2 2023-08-10T06:29:40,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@181659c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:29:40,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:29:40,963 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:53436, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:29:40,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:29:40,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:29:40,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=91, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1691648980955_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } 2023-08-10T06:29:40,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1691648980955_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 }, snapshot procedure id = 91 2023-08-10T06:29:40,967 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1691648980955_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:29:40,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=91 2023-08-10T06:29:40,967 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1691648980955_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:29:40,969 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1691648980955_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:29:41,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=91 2023-08-10T06:29:41,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=91 2023-08-10T06:29:41,375 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1691648980955_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:29:41,375 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE; SnapshotRegionProcedure 4897eee4660ec19a1cc212f00aac000c}] 2023-08-10T06:29:41,377 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=92, ppid=91, state=RUNNABLE; SnapshotRegionProcedure 4897eee4660ec19a1cc212f00aac000c 2023-08-10T06:29:41,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45101 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=92 2023-08-10T06:29:41,528 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-16916488687341,,1691648901068.4897eee4660ec19a1cc212f00aac000c. 2023-08-10T06:29:41,529 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] regionserver.HRegion(2520): Flush status journal for 4897eee4660ec19a1cc212f00aac000c: 2023-08-10T06:29:41,529 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-16916488687341,,1691648901068.4897eee4660ec19a1cc212f00aac000c. for snapshot_1691648980955_ns2_test-16916488687341 completed. 2023-08-10T06:29:41,529 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] snapshot.SnapshotManifest(240): Storing 'ns2:test-16916488687341,,1691648901068.4897eee4660ec19a1cc212f00aac000c.' region-info for snapshot=snapshot_1691648980955_ns2_test-16916488687341 2023-08-10T06:29:41,529 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:29:41,529 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/4897eee4660ec19a1cc212f00aac000c/f/a272e7ab9ac64bbb812d9b65df841988_SeqId_4_] hfiles 2023-08-10T06:29:41,529 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/4897eee4660ec19a1cc212f00aac000c/f/a272e7ab9ac64bbb812d9b65df841988_SeqId_4_ for snapshot=snapshot_1691648980955_ns2_test-16916488687341 2023-08-10T06:29:41,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=91 2023-08-10T06:29:41,936 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-16916488687341,,1691648901068.4897eee4660ec19a1cc212f00aac000c. 2023-08-10T06:29:41,936 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=92 2023-08-10T06:29:41,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster(3994): Remote procedure done, pid=92 2023-08-10T06:29:41,936 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1691648980955_ns2_test-16916488687341 on region 4897eee4660ec19a1cc212f00aac000c 2023-08-10T06:29:41,937 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=92, ppid=91, state=RUNNABLE; SnapshotRegionProcedure 4897eee4660ec19a1cc212f00aac000c 2023-08-10T06:29:41,939 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=92, resume processing ppid=91 2023-08-10T06:29:41,940 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=92, ppid=91, state=SUCCESS; SnapshotRegionProcedure 4897eee4660ec19a1cc212f00aac000c in 563 msec 2023-08-10T06:29:41,940 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1691648980955_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:29:41,940 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1691648980955_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:29:41,941 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1691648980955_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:29:41,941 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1691648980955_ns2_test-16916488687341 2023-08-10T06:29:41,941 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_1691648980955_ns2_test-16916488687341 2023-08-10T06:29:41,956 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1691648980955_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:29:41,961 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1691648980955_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:29:41,961 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_1691648980955_ns2_test-16916488687341 to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691648980955_ns2_test-16916488687341 2023-08-10T06:29:42,007 DEBUG [regionserver/jenkins-hbase3:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.3076923076923077, tune throughput to 65.38 MB/second 2023-08-10T06:29:42,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=91 2023-08-10T06:29:42,778 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1691648980955_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:29:42,778 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1691648980955_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 }, snapshot procedure id = 91 2023-08-10T06:29:42,779 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=91, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1691648980955_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } in 1.8130 sec 2023-08-10T06:29:43,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=91 2023-08-10T06:29:43,073 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1691648983073_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } 2023-08-10T06:29:43,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691648983074 (current time:1691648983074). 2023-08-10T06:29:43,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:29:43,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1691648983073_ns3_test-16916488687342 VERSION not specified, setting to 2 2023-08-10T06:29:43,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1779bf60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:29:43,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:29:43,082 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:53450, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:29:43,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:29:43,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:29:43,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=93, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1691648983073_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } 2023-08-10T06:29:43,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1691648983073_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 }, snapshot procedure id = 93 2023-08-10T06:29:43,086 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1691648983073_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:29:43,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=93 2023-08-10T06:29:43,087 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1691648983073_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:29:43,088 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1691648983073_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:29:43,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=93 2023-08-10T06:29:43,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=93 2023-08-10T06:29:43,494 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1691648983073_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:29:43,495 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE; SnapshotRegionProcedure 77e10d7389ad2c7a8dfcbed4a1e8d4af}] 2023-08-10T06:29:43,496 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=94, ppid=93, state=RUNNABLE; SnapshotRegionProcedure 77e10d7389ad2c7a8dfcbed4a1e8d4af 2023-08-10T06:29:43,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45101 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=94 2023-08-10T06:29:43,648 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=94}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-16916488687342,,1691648905436.77e10d7389ad2c7a8dfcbed4a1e8d4af. 2023-08-10T06:29:43,649 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=94}] regionserver.HRegion(2520): Flush status journal for 77e10d7389ad2c7a8dfcbed4a1e8d4af: 2023-08-10T06:29:43,649 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=94}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-16916488687342,,1691648905436.77e10d7389ad2c7a8dfcbed4a1e8d4af. for snapshot_1691648983073_ns3_test-16916488687342 completed. 2023-08-10T06:29:43,649 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=94}] snapshot.SnapshotManifest(240): Storing 'ns3:test-16916488687342,,1691648905436.77e10d7389ad2c7a8dfcbed4a1e8d4af.' region-info for snapshot=snapshot_1691648983073_ns3_test-16916488687342 2023-08-10T06:29:43,649 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=94}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:29:43,649 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=94}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-08-10T06:29:43,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=93 2023-08-10T06:29:44,055 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=94}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-16916488687342,,1691648905436.77e10d7389ad2c7a8dfcbed4a1e8d4af. 2023-08-10T06:29:44,055 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=94 2023-08-10T06:29:44,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster(3994): Remote procedure done, pid=94 2023-08-10T06:29:44,055 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1691648983073_ns3_test-16916488687342 on region 77e10d7389ad2c7a8dfcbed4a1e8d4af 2023-08-10T06:29:44,057 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=94, ppid=93, state=RUNNABLE; SnapshotRegionProcedure 77e10d7389ad2c7a8dfcbed4a1e8d4af 2023-08-10T06:29:44,059 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=94, resume processing ppid=93 2023-08-10T06:29:44,059 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=94, ppid=93, state=SUCCESS; SnapshotRegionProcedure 77e10d7389ad2c7a8dfcbed4a1e8d4af in 563 msec 2023-08-10T06:29:44,059 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1691648983073_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:29:44,059 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1691648983073_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:29:44,060 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1691648983073_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:29:44,060 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1691648983073_ns3_test-16916488687342 2023-08-10T06:29:44,060 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_1691648983073_ns3_test-16916488687342 2023-08-10T06:29:44,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=93 2023-08-10T06:29:44,469 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1691648983073_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:29:44,474 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1691648983073_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:29:44,475 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_1691648983073_ns3_test-16916488687342 to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691648983073_ns3_test-16916488687342 2023-08-10T06:29:45,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=93 2023-08-10T06:29:45,293 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1691648983073_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:29:45,293 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1691648983073_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 }, snapshot procedure id = 93 2023-08-10T06:29:45,294 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=93, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1691648983073_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } in 2.2090 sec 2023-08-10T06:29:47,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=93 2023-08-10T06:29:47,193 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1691648976628 2023-08-10T06:29:47,193 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-08-10T06:29:47,193 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2023-08-10T06:29:47,193 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1691648976628_ns3:test-16916488687342 2023-08-10T06:29:47,193 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1691648983073_ns3_test-16916488687342 to hdfs://localhost:45379/backupUT/backup_1691648976628/ns3/test-16916488687342/ 2023-08-10T06:29:47,193 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-08-10T06:29:47,216 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:45379, inputRoot=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e 2023-08-10T06:29:47,216 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,33705,1691648856595_-1229582583_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:45379/backupUT/backup_1691648976628/ns3/test-16916488687342, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:45379/backupUT/backup_1691648976628/ns3/test-16916488687342/.hbase-snapshot/.tmp/snapshot_1691648983073_ns3_test-16916488687342 2023-08-10T06:29:47,216 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:45379, inputRoot=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e, snapshotDir=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691648983073_ns3_test-16916488687342. 2023-08-10T06:29:47,222 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691648983073_ns3_test-16916488687342 to hdfs://localhost:45379/backupUT/backup_1691648976628/ns3/test-16916488687342/.hbase-snapshot/.tmp/snapshot_1691648983073_ns3_test-16916488687342 2023-08-10T06:29:47,637 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:47,638 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:47,638 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:47,639 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:48,637 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/hadoop-2153141607811990682.jar 2023-08-10T06:29:48,637 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:48,638 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:48,638 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:48,639 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:48,639 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:48,639 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:48,639 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:48,640 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:48,640 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:48,641 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:29:48,641 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-08-10T06:29:48,642 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-08-10T06:29:48,642 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-08-10T06:29:48,643 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-08-10T06:29:48,643 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-08-10T06:29:48,643 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-08-10T06:29:48,643 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-08-10T06:29:48,644 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-08-10T06:29:48,644 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-08-10T06:29:48,645 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-08-10T06:29:48,645 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-08-10T06:29:48,646 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:29:48,646 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:29:48,647 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:29:48,647 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:29:48,647 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:29:48,648 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:29:48,648 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:29:48,691 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1691648983073_ns3_test-16916488687342' hfile list 2023-08-10T06:29:48,761 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691648988728/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-08-10T06:29:48,761 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691648988728/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-08-10T06:29:49,790 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-08-10T06:29:49,791 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-08-10T06:29:49,796 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1691648983073_ns3_test-16916488687342 2023-08-10T06:29:49,796 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1691648983073_ns3_test-16916488687342 finished. 2023-08-10T06:29:49,796 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1691648976628_ns2:test-16916488687341 2023-08-10T06:29:49,796 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1691648980955_ns2_test-16916488687341 to hdfs://localhost:45379/backupUT/backup_1691648976628/ns2/test-16916488687341/ 2023-08-10T06:29:49,796 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-08-10T06:29:49,819 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:45379, inputRoot=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e 2023-08-10T06:29:49,819 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,33705,1691648856595_201440096_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:45379/backupUT/backup_1691648976628/ns2/test-16916488687341, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:45379/backupUT/backup_1691648976628/ns2/test-16916488687341/.hbase-snapshot/.tmp/snapshot_1691648980955_ns2_test-16916488687341 2023-08-10T06:29:49,819 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:45379, inputRoot=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e, snapshotDir=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691648980955_ns2_test-16916488687341. 2023-08-10T06:29:49,826 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691648980955_ns2_test-16916488687341 to hdfs://localhost:45379/backupUT/backup_1691648976628/ns2/test-16916488687341/.hbase-snapshot/.tmp/snapshot_1691648980955_ns2_test-16916488687341 2023-08-10T06:29:50,248 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:50,248 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:50,249 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:50,249 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:51,278 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/hadoop-7324265543583018259.jar 2023-08-10T06:29:51,278 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:51,278 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:51,279 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:51,279 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:51,279 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:51,280 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:51,280 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:51,280 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:51,281 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-08-10T06:29:51,281 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:29:51,282 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-08-10T06:29:51,282 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-08-10T06:29:51,283 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-08-10T06:29:51,283 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-08-10T06:29:51,284 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-08-10T06:29:51,284 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-08-10T06:29:51,284 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-08-10T06:29:51,285 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-08-10T06:29:51,285 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-08-10T06:29:51,285 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-08-10T06:29:51,286 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-08-10T06:29:51,286 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:29:51,287 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:29:51,287 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:29:51,288 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:29:51,288 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:29:51,289 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:29:51,289 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:29:51,332 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1691648980955_ns2_test-16916488687341' hfile list 2023-08-10T06:29:51,334 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-08-10T06:29:51,400 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691648991368/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-08-10T06:29:51,400 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691648991368/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-08-10T06:29:51,451 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-08-10T06:29:51,476 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/4897eee4660ec19a1cc212f00aac000c/f/a272e7ab9ac64bbb812d9b65df841988_SeqId_4_ output=hdfs://localhost:45379/backupUT/backup_1691648976628/ns2/test-16916488687341/archive/data/ns2/test-16916488687341/4897eee4660ec19a1cc212f00aac000c/f/a272e7ab9ac64bbb812d9b65df841988_SeqId_4_ 2023-08-10T06:29:51,476 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8725 (8.5 K) time=0sec 8.321M/sec 2023-08-10T06:29:52,435 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-08-10T06:29:52,436 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-08-10T06:29:52,442 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1691648980955_ns2_test-16916488687341 2023-08-10T06:29:52,442 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1691648980955_ns2_test-16916488687341 finished. 2023-08-10T06:29:52,442 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(1014): ns3:test-16916488687342 2023-08-10T06:29:52,442 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(1014): ns2:test-16916488687341 2023-08-10T06:29:52,449 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691648976628 2023-08-10T06:29:52,449 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-08-10T06:29:52,854 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:45379/backupUT/backup_1691648976628/.backup.manifest 2023-08-10T06:29:52,854 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691648976628 2023-08-10T06:29:52,854 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-08-10T06:29:53,259 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:45379/backupUT/backup_1691648976628/.backup.manifest 2023-08-10T06:29:53,259 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(358): Backup backup_1691648976628 finished: type=FULL,tablelist=ns3:test-16916488687342;ns2:test-16916488687341,targetRootDir=hdfs://localhost:45379/backupUT,startts=1691648980770,completets=1691648992448,bytescopied=0 2023-08-10T06:29:53,259 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-08-10T06:29:53,259 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1691648983073_ns3_test-16916488687342 2023-08-10T06:29:53,260 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1691648983073_ns3_test-16916488687342" 2023-08-10T06:29:53,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1691648983073_ns3_test-16916488687342 2023-08-10T06:29:53,263 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1691648983073_ns3_test-16916488687342 for backup backup_1691648976628 succeeded. 2023-08-10T06:29:53,263 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1691648980955_ns2_test-16916488687341 2023-08-10T06:29:53,265 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1691648980955_ns2_test-16916488687341" 2023-08-10T06:29:53,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1691648980955_ns2_test-16916488687341 2023-08-10T06:29:53,268 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1691648980955_ns2_test-16916488687341 for backup backup_1691648976628 succeeded. 2023-08-10T06:29:53,268 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-08-10T06:29:53,272 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-08-10T06:29:53,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-08-10T06:29:53,274 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-08-10T06:29:53,276 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-08-10T06:29:53,282 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(376): Backup backup_1691648976628 completed. 2023-08-10T06:29:53,282 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(513): Stopping rpc client You have to specify either set name or table list to restore Usage: hbase restore [options] backup_path Path to a backup destination root backup_id Backup image ID to restore table(s) Comma-separated list of tables to restore Options: -c Check restore sequence and dependencies only (does not execute the command) -d Enable debug loggings -h,--help Show usage -m A comma separated list of target tables. If specified, each table in must have a mapping -o Overwrite data if any of the restore target tables exists -q Yarn queue name to run backup restore command on -s Backup set to restore, mutually exclusive with -t (table list) -t Table name list, comma-separated. To enable backup, in hbase-site.xml, set: hbase.backup.enable=true hbase.master.logcleaner.plugins=YOUR_PLUGINS,org.apache.hadoop.hbase.backup.master.BackupLogCleaner hbase.procedure.master.classes=YOUR_CLASSES,org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager hbase.procedure.regionserver.classes=YOUR_CLASSES,org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager hbase.coprocessor.region.classes=YOUR_CLASSES,org.apache.hadoop.hbase.backup.BackupObserver and restart the cluster For more information please see http://hbase.apache.org/book.html#backuprestore 2023-08-10T06:29:53,314 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreMultipleDNECommand Thread=597 (was 604), OpenFileDescriptor=679 (was 688), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=93 (was 119), ProcessCount=169 (was 169), AvailableMemoryMB=1384 (was 1474) 2023-08-10T06:29:53,314 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(130): Thread=597 is superior to 500 2023-08-10T06:29:53,331 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreMultiple Thread=597, OpenFileDescriptor=679, MaxFileDescriptor=60000, SystemLoadAverage=93, ProcessCount=169, AvailableMemoryMB=1383 2023-08-10T06:29:53,331 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(130): Thread=597 is superior to 500 2023-08-10T06:29:53,331 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] backup.TestFullRestore(120): create full backup image on multiple tables 2023-08-10T06:29:53,339 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40fe759b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:29:53,345 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:29:53,347 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:48336, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:29:53,349 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:29:53,350 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:53782, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:29:53,360 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-08-10T06:29:53,366 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:29:53,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691648993366 (current time:1691648993366). 2023-08-10T06:29:53,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:29:53,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-08-10T06:29:53,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c1303e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:29:53,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:29:53,375 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:48352, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:29:53,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:29:53,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:29:53,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=95, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:29:53,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 95 2023-08-10T06:29:53,379 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:29:53,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=95 2023-08-10T06:29:53,379 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:29:53,381 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:29:53,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=95 2023-08-10T06:29:53,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=95 2023-08-10T06:29:53,787 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:29:53,787 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72}] 2023-08-10T06:29:53,789 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=96, ppid=95, state=RUNNABLE; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:29:53,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45101 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=96 2023-08-10T06:29:53,941 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. 2023-08-10T06:29:53,941 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.HRegion(2819): Flushing b9c493f2c9fbef45de19e8268a534e72 2/2 column families, dataSize=1.50 KB heapSize=2.73 KB 2023-08-10T06:29:53,944 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/meta/0ae449898702485faa63081b0f3f028a is 171, key is trslm:hdfs://localhost:45379/backupUT\x00ns2:test-16916488687341/meta:log-roll-map/1691648992445/Put/seqid=0 2023-08-10T06:29:53,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=95 2023-08-10T06:29:54,350 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=717 B at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/meta/0ae449898702485faa63081b0f3f028a 2023-08-10T06:29:54,355 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 0ae449898702485faa63081b0f3f028a 2023-08-10T06:29:54,357 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/session/40f9fbdeac5b49dbb654e429c177708d is 524, key is session:backup_1691648976628/session:context/1691648993275/Put/seqid=0 2023-08-10T06:29:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=95 2023-08-10T06:29:54,763 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=820 B at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/session/40f9fbdeac5b49dbb654e429c177708d 2023-08-10T06:29:54,767 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 40f9fbdeac5b49dbb654e429c177708d 2023-08-10T06:29:54,768 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/meta/0ae449898702485faa63081b0f3f028a as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/0ae449898702485faa63081b0f3f028a 2023-08-10T06:29:54,773 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/0ae449898702485faa63081b0f3f028a, entries=6, sequenceid=83, filesize=5.8 K 2023-08-10T06:29:54,774 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/session/40f9fbdeac5b49dbb654e429c177708d as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/40f9fbdeac5b49dbb654e429c177708d 2023-08-10T06:29:54,778 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/40f9fbdeac5b49dbb654e429c177708d, entries=2, sequenceid=83, filesize=5.5 K 2023-08-10T06:29:54,779 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.HRegion(3022): Finished flush of dataSize ~1.50 KB/1537, heapSize ~2.70 KB/2760, currentSize=0 B/0 for b9c493f2c9fbef45de19e8268a534e72 in 838ms, sequenceid=83, compaction requested=true 2023-08-10T06:29:54,779 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.HRegion(2520): Flush status journal for b9c493f2c9fbef45de19e8268a534e72: 2023-08-10T06:29:54,779 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. for snapshot_backup_system completed. 2023-08-10T06:29:54,779 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72.' region-info for snapshot=snapshot_backup_system 2023-08-10T06:29:54,779 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:29:54,779 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/bc10682e7f36491d941e2fe62e2e4100, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/9d9a4feb1c4e44d99cc63e3f68878731, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/f04a99990b2c429eaf1096ef413be3b8, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/9e97dc38b66b476f952990b23fa0a3ca, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/930c578fe3f543e1b6308abfb31c7246, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/24713cefebe0499c812c6d209dea0fd6, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/0ae449898702485faa63081b0f3f028a] hfiles 2023-08-10T06:29:54,779 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (1/7): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/bc10682e7f36491d941e2fe62e2e4100 for snapshot=snapshot_backup_system 2023-08-10T06:29:54,779 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (2/7): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/9d9a4feb1c4e44d99cc63e3f68878731 for snapshot=snapshot_backup_system 2023-08-10T06:29:54,779 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (3/7): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/f04a99990b2c429eaf1096ef413be3b8 for snapshot=snapshot_backup_system 2023-08-10T06:29:54,779 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (4/7): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/9e97dc38b66b476f952990b23fa0a3ca for snapshot=snapshot_backup_system 2023-08-10T06:29:54,779 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (5/7): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/930c578fe3f543e1b6308abfb31c7246 for snapshot=snapshot_backup_system 2023-08-10T06:29:54,779 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (6/7): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/24713cefebe0499c812c6d209dea0fd6 for snapshot=snapshot_backup_system 2023-08-10T06:29:54,779 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (7/7): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/0ae449898702485faa63081b0f3f028a for snapshot=snapshot_backup_system 2023-08-10T06:29:54,779 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/e88678d5b4b14eecabac817daf12539e, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/37561b9a8ae2406bb6d053c33aeff31c, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/883911c19cab473f8bf801f934c90bc1, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/0c7705a45c554d6a97fa37fac319ceff, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/398a15bfffb84acfadb1be252c2e133a, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/adceb6320b02451abd02beed6c69c000, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/68fd0cf6c3094cc7af1a5176829fd9a4, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/40f9fbdeac5b49dbb654e429c177708d] hfiles 2023-08-10T06:29:54,779 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (1/8): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/e88678d5b4b14eecabac817daf12539e for snapshot=snapshot_backup_system 2023-08-10T06:29:54,780 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (2/8): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/37561b9a8ae2406bb6d053c33aeff31c for snapshot=snapshot_backup_system 2023-08-10T06:29:54,780 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (3/8): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/883911c19cab473f8bf801f934c90bc1 for snapshot=snapshot_backup_system 2023-08-10T06:29:54,780 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (4/8): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/0c7705a45c554d6a97fa37fac319ceff for snapshot=snapshot_backup_system 2023-08-10T06:29:54,780 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (5/8): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/398a15bfffb84acfadb1be252c2e133a for snapshot=snapshot_backup_system 2023-08-10T06:29:54,780 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (6/8): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/adceb6320b02451abd02beed6c69c000 for snapshot=snapshot_backup_system 2023-08-10T06:29:54,780 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (7/8): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/68fd0cf6c3094cc7af1a5176829fd9a4 for snapshot=snapshot_backup_system 2023-08-10T06:29:54,780 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (8/8): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/40f9fbdeac5b49dbb654e429c177708d for snapshot=snapshot_backup_system 2023-08-10T06:29:55,185 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. 2023-08-10T06:29:55,185 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=96 2023-08-10T06:29:55,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster(3994): Remote procedure done, pid=96 2023-08-10T06:29:55,185 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:29:55,189 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=96, ppid=95, state=RUNNABLE; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:29:55,191 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=96, resume processing ppid=95 2023-08-10T06:29:55,191 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=96, ppid=95, state=SUCCESS; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72 in 1.4030 sec 2023-08-10T06:29:55,191 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:29:55,192 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:29:55,193 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:29:55,193 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-08-10T06:29:55,194 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_backup_system 2023-08-10T06:29:55,211 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:29:55,224 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:29:55,225 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_backup_system 2023-08-10T06:29:55,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=95 2023-08-10T06:29:56,048 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:29:56,048 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 95 2023-08-10T06:29:56,049 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=95, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.6710 sec 2023-08-10T06:29:57,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=95 2023-08-10T06:29:57,486 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(115): Backup backup_1691648993339 started at 1691648997485. 2023-08-10T06:29:57,487 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(119): Backup session backup_1691648993339 has been started. 2023-08-10T06:29:57,492 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-08-10T06:29:57,492 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-08-10T06:29:57,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure.ProcedureCoordinator(143): Procedure rolllog was in running list but was completed. Accepting new attempt. 2023-08-10T06:29:57,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-08-10T06:29:57,493 INFO [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-08-10T06:29:57,493 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-08-10T06:29:57,494 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-08-10T06:29:57,494 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-08-10T06:29:57,494 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:29:57,500 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:57,500 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:29:57,500 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:29:57,501 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-08-10T06:29:57,501 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-08-10T06:29:57,501 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-08-10T06:29:57,501 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:29:57,501 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-08-10T06:29:57,502 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-08-10T06:29:57,502 WARN [zk-event-processor-pool-0 {}] procedure.ProcedureMember(133): A completed old subproc rolllog is still present, removing 2023-08-10T06:29:57,504 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-08-10T06:29:57,504 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-08-10T06:29:57,504 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-08-10T06:29:57,505 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-08-10T06:29:57,505 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-08-10T06:29:57,505 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,45101,1691648859755' joining acquired barrier for procedure (rolllog) in zk 2023-08-10T06:29:57,507 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-08-10T06:29:57,507 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:57,507 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:29:57,507 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:29:57,508 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-08-10T06:29:57,508 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:29:57,508 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:29:57,508 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:29:57,508 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:57,509 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:29:57,509 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,45101,1691648859755' joining acquired barrier for procedure 'rolllog' on coordinator 2023-08-10T06:29:57,509 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@37633be7[Count = 0] remaining members to acquire global barrier 2023-08-10T06:29:57,509 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-08-10T06:29:57,509 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-08-10T06:29:57,510 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-08-10T06:29:57,510 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-08-10T06:29:57,510 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-08-10T06:29:57,510 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-08-10T06:29:57,510 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-08-10T06:29:57,511 DEBUG [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:57,511 INFO [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1691648980808 highest: 1691648980808 on jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:57,511 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta:.meta(num 1691648980788) roll requested 2023-08-10T06:29:57,522 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691648997511.meta, exclude list is [], retry=0 2023-08-10T06:29:57,528 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK] 2023-08-10T06:29:57,529 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691648980788.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691648997511.meta 2023-08-10T06:29:57,529 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK]] 2023-08-10T06:29:57,529 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691648980788.meta is not closed yet, will try archiving it next time 2023-08-10T06:29:57,529 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C45101%2C1691648859755:(num 1691648980808) roll requested 2023-08-10T06:29:57,531 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691648980788.meta not finished, retry = 0 2023-08-10T06:29:57,540 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691648997529, exclude list is [], retry=0 2023-08-10T06:29:57,543 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK] 2023-08-10T06:29:57,545 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691648980808 with entries=10, filesize=2.67 KB; new WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691648997529 2023-08-10T06:29:57,545 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK]] 2023-08-10T06:29:57,545 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691648980808 is not closed yet, will try archiving it next time 2023-08-10T06:29:57,551 DEBUG [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-08-10T06:29:57,551 INFO [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1691648997529 on jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:57,558 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-08-10T06:29:57,558 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,45101,1691648859755' in zk 2023-08-10T06:29:57,561 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-08-10T06:29:57,561 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:29:57,561 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-08-10T06:29:57,561 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:57,563 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:29:57,563 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:29:57,563 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:29:57,564 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:29:57,564 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:29:57,564 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:57,565 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:29:57,565 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:29:57,566 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:57,566 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,45101,1691648859755': 2023-08-10T06:29:57,566 INFO [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-08-10T06:29:57,566 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,45101,1691648859755' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-08-10T06:29:57,567 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-08-10T06:29:57,567 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-08-10T06:29:57,567 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-08-10T06:29:57,567 INFO [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-08-10T06:29:57,569 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-08-10T06:29:57,569 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-08-10T06:29:57,570 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:29:57,570 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:29:57,570 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:29:57,570 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:29:57,570 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:29:57,570 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:29:57,570 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:29:57,571 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:29:57,571 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:29:57,571 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:29:57,572 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:57,572 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:29:57,572 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:29:57,573 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:29:57,576 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:29:57,576 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:29:57,576 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:29:57,576 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-08-10T06:29:57,576 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-08-10T06:29:57,577 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:29:57,577 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:29:57,633 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691648980788.meta to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/oldWALs/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691648980788.meta 2023-08-10T06:29:57,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-08-10T06:29:57,681 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1691648997680_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } 2023-08-10T06:29:57,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691648997681 (current time:1691648997681). 2023-08-10T06:29:57,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:29:57,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1691648997680_ns2_test-16916488687341 VERSION not specified, setting to 2 2023-08-10T06:29:57,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32b2d63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:29:57,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:29:57,690 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:48360, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:29:57,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:29:57,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:29:57,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=97, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1691648997680_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } 2023-08-10T06:29:57,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1691648997680_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 }, snapshot procedure id = 97 2023-08-10T06:29:57,694 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1691648997680_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:29:57,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=97 2023-08-10T06:29:57,695 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1691648997680_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:29:57,696 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1691648997680_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:29:57,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=97 2023-08-10T06:29:57,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=97 2023-08-10T06:29:58,101 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1691648997680_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:29:58,101 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE; SnapshotRegionProcedure 4897eee4660ec19a1cc212f00aac000c}] 2023-08-10T06:29:58,103 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=98, ppid=97, state=RUNNABLE; SnapshotRegionProcedure 4897eee4660ec19a1cc212f00aac000c 2023-08-10T06:29:58,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45101 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=98 2023-08-10T06:29:58,255 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-16916488687341,,1691648901068.4897eee4660ec19a1cc212f00aac000c. 2023-08-10T06:29:58,255 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] regionserver.HRegion(2520): Flush status journal for 4897eee4660ec19a1cc212f00aac000c: 2023-08-10T06:29:58,255 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-16916488687341,,1691648901068.4897eee4660ec19a1cc212f00aac000c. for snapshot_1691648997680_ns2_test-16916488687341 completed. 2023-08-10T06:29:58,255 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] snapshot.SnapshotManifest(240): Storing 'ns2:test-16916488687341,,1691648901068.4897eee4660ec19a1cc212f00aac000c.' region-info for snapshot=snapshot_1691648997680_ns2_test-16916488687341 2023-08-10T06:29:58,255 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:29:58,255 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/4897eee4660ec19a1cc212f00aac000c/f/a272e7ab9ac64bbb812d9b65df841988_SeqId_4_] hfiles 2023-08-10T06:29:58,255 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/4897eee4660ec19a1cc212f00aac000c/f/a272e7ab9ac64bbb812d9b65df841988_SeqId_4_ for snapshot=snapshot_1691648997680_ns2_test-16916488687341 2023-08-10T06:29:58,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=97 2023-08-10T06:29:58,662 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-16916488687341,,1691648901068.4897eee4660ec19a1cc212f00aac000c. 2023-08-10T06:29:58,662 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=98 2023-08-10T06:29:58,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster(3994): Remote procedure done, pid=98 2023-08-10T06:29:58,662 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1691648997680_ns2_test-16916488687341 on region 4897eee4660ec19a1cc212f00aac000c 2023-08-10T06:29:58,664 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=98, ppid=97, state=RUNNABLE; SnapshotRegionProcedure 4897eee4660ec19a1cc212f00aac000c 2023-08-10T06:29:58,665 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=98, resume processing ppid=97 2023-08-10T06:29:58,665 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=98, ppid=97, state=SUCCESS; SnapshotRegionProcedure 4897eee4660ec19a1cc212f00aac000c in 563 msec 2023-08-10T06:29:58,665 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1691648997680_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:29:58,666 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1691648997680_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:29:58,666 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1691648997680_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:29:58,666 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1691648997680_ns2_test-16916488687341 2023-08-10T06:29:58,668 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_1691648997680_ns2_test-16916488687341 2023-08-10T06:29:58,676 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1691648997680_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:29:58,680 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1691648997680_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:29:58,681 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_1691648997680_ns2_test-16916488687341 to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691648997680_ns2_test-16916488687341 2023-08-10T06:29:58,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=97 2023-08-10T06:29:59,498 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1691648997680_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:29:59,498 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1691648997680_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 }, snapshot procedure id = 97 2023-08-10T06:29:59,499 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=97, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1691648997680_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } in 1.8060 sec 2023-08-10T06:29:59,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=97 2023-08-10T06:29:59,799 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1691648999799_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } 2023-08-10T06:29:59,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691648999799 (current time:1691648999799). 2023-08-10T06:29:59,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:29:59,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1691648999799_ns3_test-16916488687342 VERSION not specified, setting to 2 2023-08-10T06:29:59,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6eb622c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:29:59,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:29:59,807 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:48376, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:29:59,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:29:59,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:29:59,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=99, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1691648999799_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } 2023-08-10T06:29:59,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1691648999799_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 }, snapshot procedure id = 99 2023-08-10T06:29:59,811 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1691648999799_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:29:59,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=99 2023-08-10T06:29:59,812 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1691648999799_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:29:59,814 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1691648999799_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:29:59,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=99 2023-08-10T06:30:00,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=99 2023-08-10T06:30:00,223 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1691648999799_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:30:00,223 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE; SnapshotRegionProcedure 77e10d7389ad2c7a8dfcbed4a1e8d4af}] 2023-08-10T06:30:00,225 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=100, ppid=99, state=RUNNABLE; SnapshotRegionProcedure 77e10d7389ad2c7a8dfcbed4a1e8d4af 2023-08-10T06:30:00,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45101 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=100 2023-08-10T06:30:00,377 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=100}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-16916488687342,,1691648905436.77e10d7389ad2c7a8dfcbed4a1e8d4af. 2023-08-10T06:30:00,377 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=100}] regionserver.HRegion(2520): Flush status journal for 77e10d7389ad2c7a8dfcbed4a1e8d4af: 2023-08-10T06:30:00,377 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=100}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-16916488687342,,1691648905436.77e10d7389ad2c7a8dfcbed4a1e8d4af. for snapshot_1691648999799_ns3_test-16916488687342 completed. 2023-08-10T06:30:00,377 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=100}] snapshot.SnapshotManifest(240): Storing 'ns3:test-16916488687342,,1691648905436.77e10d7389ad2c7a8dfcbed4a1e8d4af.' region-info for snapshot=snapshot_1691648999799_ns3_test-16916488687342 2023-08-10T06:30:00,377 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=100}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:30:00,377 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=100}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-08-10T06:30:00,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=99 2023-08-10T06:30:00,783 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=100}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-16916488687342,,1691648905436.77e10d7389ad2c7a8dfcbed4a1e8d4af. 2023-08-10T06:30:00,783 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=100 2023-08-10T06:30:00,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster(3994): Remote procedure done, pid=100 2023-08-10T06:30:00,783 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1691648999799_ns3_test-16916488687342 on region 77e10d7389ad2c7a8dfcbed4a1e8d4af 2023-08-10T06:30:00,784 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=100, ppid=99, state=RUNNABLE; SnapshotRegionProcedure 77e10d7389ad2c7a8dfcbed4a1e8d4af 2023-08-10T06:30:00,786 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=100, resume processing ppid=99 2023-08-10T06:30:00,786 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=100, ppid=99, state=SUCCESS; SnapshotRegionProcedure 77e10d7389ad2c7a8dfcbed4a1e8d4af in 562 msec 2023-08-10T06:30:00,786 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1691648999799_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:30:00,787 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1691648999799_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:30:00,787 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1691648999799_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:30:00,787 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1691648999799_ns3_test-16916488687342 2023-08-10T06:30:00,789 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_1691648999799_ns3_test-16916488687342 2023-08-10T06:30:00,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=99 2023-08-10T06:30:01,198 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1691648999799_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:30:01,202 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1691648999799_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:30:01,203 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_1691648999799_ns3_test-16916488687342 to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691648999799_ns3_test-16916488687342 2023-08-10T06:30:01,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=99 2023-08-10T06:30:02,019 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1691648999799_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:30:02,019 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1691648999799_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 }, snapshot procedure id = 99 2023-08-10T06:30:02,020 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=99, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1691648999799_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } in 2.2100 sec 2023-08-10T06:30:03,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=99 2023-08-10T06:30:03,917 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1691648993339 2023-08-10T06:30:03,917 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-08-10T06:30:03,918 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2023-08-10T06:30:03,918 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1691648993339_ns3:test-16916488687342 2023-08-10T06:30:03,918 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1691648999799_ns3_test-16916488687342 to hdfs://localhost:45379/backupUT/backup_1691648993339/ns3/test-16916488687342/ 2023-08-10T06:30:03,918 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-08-10T06:30:03,940 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:45379, inputRoot=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e 2023-08-10T06:30:03,940 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,33705,1691648856595_634808638_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:45379/backupUT/backup_1691648993339/ns3/test-16916488687342, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:45379/backupUT/backup_1691648993339/ns3/test-16916488687342/.hbase-snapshot/.tmp/snapshot_1691648999799_ns3_test-16916488687342 2023-08-10T06:30:03,940 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:45379, inputRoot=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e, snapshotDir=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691648999799_ns3_test-16916488687342. 2023-08-10T06:30:03,947 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691648999799_ns3_test-16916488687342 to hdfs://localhost:45379/backupUT/backup_1691648993339/ns3/test-16916488687342/.hbase-snapshot/.tmp/snapshot_1691648999799_ns3_test-16916488687342 2023-08-10T06:30:04,364 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:04,365 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:04,365 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:04,365 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:05,324 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/hadoop-4711090481841955544.jar 2023-08-10T06:30:05,324 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:05,324 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:05,325 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:05,325 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:05,326 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:05,326 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:05,326 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:05,327 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:05,327 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:05,328 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:30:05,328 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-08-10T06:30:05,329 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-08-10T06:30:05,329 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-08-10T06:30:05,329 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-08-10T06:30:05,330 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-08-10T06:30:05,330 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-08-10T06:30:05,330 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-08-10T06:30:05,331 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-08-10T06:30:05,331 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-08-10T06:30:05,332 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-08-10T06:30:05,332 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-08-10T06:30:05,332 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:30:05,333 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:30:05,333 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:30:05,334 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:30:05,334 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:30:05,335 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:30:05,335 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:30:05,379 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1691648999799_ns3_test-16916488687342' hfile list 2023-08-10T06:30:05,454 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691649005419/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-08-10T06:30:05,454 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691649005419/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-08-10T06:30:06,482 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-08-10T06:30:06,483 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-08-10T06:30:06,487 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1691648999799_ns3_test-16916488687342 2023-08-10T06:30:06,487 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1691648999799_ns3_test-16916488687342 finished. 2023-08-10T06:30:06,487 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1691648993339_ns2:test-16916488687341 2023-08-10T06:30:06,487 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1691648997680_ns2_test-16916488687341 to hdfs://localhost:45379/backupUT/backup_1691648993339/ns2/test-16916488687341/ 2023-08-10T06:30:06,488 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-08-10T06:30:06,510 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:45379, inputRoot=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e 2023-08-10T06:30:06,510 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,33705,1691648856595_249376165_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:45379/backupUT/backup_1691648993339/ns2/test-16916488687341, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:45379/backupUT/backup_1691648993339/ns2/test-16916488687341/.hbase-snapshot/.tmp/snapshot_1691648997680_ns2_test-16916488687341 2023-08-10T06:30:06,510 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:45379, inputRoot=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e, snapshotDir=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691648997680_ns2_test-16916488687341. 2023-08-10T06:30:06,517 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691648997680_ns2_test-16916488687341 to hdfs://localhost:45379/backupUT/backup_1691648993339/ns2/test-16916488687341/.hbase-snapshot/.tmp/snapshot_1691648997680_ns2_test-16916488687341 2023-08-10T06:30:06,931 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:06,931 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:06,932 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:06,932 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:07,891 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/hadoop-1947787634673715320.jar 2023-08-10T06:30:07,892 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:07,892 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:07,892 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:07,893 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:07,893 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:07,894 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:07,894 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:07,894 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:07,894 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:07,895 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:30:07,896 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-08-10T06:30:07,896 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-08-10T06:30:07,897 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-08-10T06:30:07,897 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-08-10T06:30:07,898 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-08-10T06:30:07,898 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-08-10T06:30:07,898 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-08-10T06:30:07,899 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-08-10T06:30:07,899 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-08-10T06:30:07,900 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-08-10T06:30:07,900 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-08-10T06:30:07,900 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:30:07,901 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:30:07,901 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:30:07,902 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:30:07,902 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:30:07,903 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:30:07,903 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:30:07,946 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1691648997680_ns2_test-16916488687341' hfile list 2023-08-10T06:30:07,948 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-08-10T06:30:08,018 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691649007983/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-08-10T06:30:08,019 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691649007983/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-08-10T06:30:08,071 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-08-10T06:30:08,124 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/4897eee4660ec19a1cc212f00aac000c/f/a272e7ab9ac64bbb812d9b65df841988_SeqId_4_ output=hdfs://localhost:45379/backupUT/backup_1691648993339/ns2/test-16916488687341/archive/data/ns2/test-16916488687341/4897eee4660ec19a1cc212f00aac000c/f/a272e7ab9ac64bbb812d9b65df841988_SeqId_4_ 2023-08-10T06:30:08,124 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8725 (8.5 K) time=0sec 8.321M/sec 2023-08-10T06:30:09,054 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-08-10T06:30:09,055 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-08-10T06:30:09,061 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1691648997680_ns2_test-16916488687341 2023-08-10T06:30:09,061 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1691648997680_ns2_test-16916488687341 finished. 2023-08-10T06:30:09,061 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(1014): ns3:test-16916488687342 2023-08-10T06:30:09,061 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(1014): ns2:test-16916488687341 2023-08-10T06:30:09,068 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691648993339 2023-08-10T06:30:09,068 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-08-10T06:30:09,473 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:45379/backupUT/backup_1691648993339/.backup.manifest 2023-08-10T06:30:09,473 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691648993339 2023-08-10T06:30:09,473 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-08-10T06:30:09,879 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:45379/backupUT/backup_1691648993339/.backup.manifest 2023-08-10T06:30:09,879 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(358): Backup backup_1691648993339 finished: type=FULL,tablelist=ns3:test-16916488687342;ns2:test-16916488687341,targetRootDir=hdfs://localhost:45379/backupUT,startts=1691648997485,completets=1691649009068,bytescopied=0 2023-08-10T06:30:09,879 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-08-10T06:30:09,879 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1691648999799_ns3_test-16916488687342 2023-08-10T06:30:09,880 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1691648999799_ns3_test-16916488687342" 2023-08-10T06:30:09,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1691648999799_ns3_test-16916488687342 2023-08-10T06:30:09,883 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1691648999799_ns3_test-16916488687342 for backup backup_1691648993339 succeeded. 2023-08-10T06:30:09,883 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1691648997680_ns2_test-16916488687341 2023-08-10T06:30:09,883 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1691648997680_ns2_test-16916488687341" 2023-08-10T06:30:09,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1691648997680_ns2_test-16916488687341 2023-08-10T06:30:09,886 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1691648997680_ns2_test-16916488687341 for backup backup_1691648993339 succeeded. 2023-08-10T06:30:09,887 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-08-10T06:30:09,890 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-08-10T06:30:09,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-08-10T06:30:09,892 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-08-10T06:30:09,894 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-08-10T06:30:09,899 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(376): Backup backup_1691648993339 completed. 2023-08-10T06:30:09,899 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:30:09,912 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(101): HBase table ns2:table2 does not exist. It will be created during restore process 2023-08-10T06:30:09,913 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(101): HBase table ns3:table3_restore does not exist. It will be created during restore process 2023-08-10T06:30:09,913 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:45379/backupUT/backup_1691648993339/.backup.manifest 2023-08-10T06:30:09,915 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691648993339/.backup.manifest 2023-08-10T06:30:09,916 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:45379/backupUT/backup_1691648993339/.backup.manifest 2023-08-10T06:30:09,917 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691648993339/.backup.manifest 2023-08-10T06:30:09,918 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:45379/backupUT/backup_1691648993339/.backup.manifest 2023-08-10T06:30:09,919 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691648993339/.backup.manifest 2023-08-10T06:30:09,919 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(148): Restoring 'ns2:test-16916488687341' to 'ns2:table2' from full backup image hdfs://localhost:45379/backupUT/backup_1691648993339/ns2/test-16916488687341 2023-08-10T06:30:09,925 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] util.RestoreTool(487): Creating target table 'ns2:table2' 2023-08-10T06:30:09,925 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] util.RestoreTool(391): Parsing region dir: hdfs://localhost:45379/backupUT/backup_1691648993339/ns2/test-16916488687341/archive/data/ns2/test-16916488687341/4897eee4660ec19a1cc212f00aac000c 2023-08-10T06:30:09,926 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:45379/backupUT/backup_1691648993339/ns2/test-16916488687341/archive/data/ns2/test-16916488687341/4897eee4660ec19a1cc212f00aac000c/f in region [hdfs://localhost:45379/backupUT/backup_1691648993339/ns2/test-16916488687341/archive/data/ns2/test-16916488687341/4897eee4660ec19a1cc212f00aac000c] 2023-08-10T06:30:09,929 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] util.RestoreTool(442): Trying to figure out region boundaries hfile=hdfs://localhost:45379/backupUT/backup_1691648993339/ns2/test-16916488687341/archive/data/ns2/test-16916488687341/4897eee4660ec19a1cc212f00aac000c/f/a272e7ab9ac64bbb812d9b65df841988_SeqId_4_ first=row0 last=row98 2023-08-10T06:30:09,929 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a272e7ab9ac64bbb812d9b65df841988_SeqId_4_ 2023-08-10T06:30:09,930 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'ns2:table2', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:30:09,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=101, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns2:table2 2023-08-10T06:30:09,932 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=101, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_PRE_OPERATION 2023-08-10T06:30:09,933 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns2" qualifier: "table2" procId is: 101 2023-08-10T06:30:09,933 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=101, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-08-10T06:30:09,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=101 2023-08-10T06:30:10,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=101 2023-08-10T06:30:10,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=101 2023-08-10T06:30:10,339 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 8212c24e55f51ac536c347dccf9f816f, NAME => 'ns2:table2,,1691649009930.8212c24e55f51ac536c347dccf9f816f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:table2', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e 2023-08-10T06:30:10,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=101 2023-08-10T06:30:10,745 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:table2,,1691649009930.8212c24e55f51ac536c347dccf9f816f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:30:10,745 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1668): Closing 8212c24e55f51ac536c347dccf9f816f, disabling compactions & flushes 2023-08-10T06:30:10,745 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:table2,,1691649009930.8212c24e55f51ac536c347dccf9f816f. 2023-08-10T06:30:10,745 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:table2,,1691649009930.8212c24e55f51ac536c347dccf9f816f. 2023-08-10T06:30:10,745 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:table2,,1691649009930.8212c24e55f51ac536c347dccf9f816f. after waiting 0 ms 2023-08-10T06:30:10,745 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:table2,,1691649009930.8212c24e55f51ac536c347dccf9f816f. 2023-08-10T06:30:10,746 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1909): Closed ns2:table2,,1691649009930.8212c24e55f51ac536c347dccf9f816f. 2023-08-10T06:30:10,746 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1622): Region close journal for 8212c24e55f51ac536c347dccf9f816f: 2023-08-10T06:30:10,747 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=101, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_ADD_TO_META 2023-08-10T06:30:10,747 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:table2,,1691649009930.8212c24e55f51ac536c347dccf9f816f.","families":{"info":[{"qualifier":"regioninfo","vlen":36,"tag":[],"timestamp":"1691649010747"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691649010747"}]},"ts":"1691649010747"} 2023-08-10T06:30:10,748 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:30:10,749 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=101, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-08-10T06:30:10,749 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649010749"}]},"ts":"1691649010749"} 2023-08-10T06:30:10,750 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:table2, state=ENABLING in hbase:meta 2023-08-10T06:30:10,756 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:table2, region=8212c24e55f51ac536c347dccf9f816f, ASSIGN}] 2023-08-10T06:30:10,758 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:table2, region=8212c24e55f51ac536c347dccf9f816f, ASSIGN 2023-08-10T06:30:10,759 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:table2, region=8212c24e55f51ac536c347dccf9f816f, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45101,1691648859755; forceNewPlan=false, retain=false 2023-08-10T06:30:10,910 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=102 updating hbase:meta row=8212c24e55f51ac536c347dccf9f816f, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:30:10,912 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=103, ppid=102, state=RUNNABLE; OpenRegionProcedure 8212c24e55f51ac536c347dccf9f816f, server=jenkins-hbase3.apache.org,45101,1691648859755}] 2023-08-10T06:30:11,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=101 2023-08-10T06:30:11,070 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] handler.AssignRegionHandler(130): Open ns2:table2,,1691649009930.8212c24e55f51ac536c347dccf9f816f. 2023-08-10T06:30:11,070 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(7254): Opening region: {ENCODED => 8212c24e55f51ac536c347dccf9f816f, NAME => 'ns2:table2,,1691649009930.8212c24e55f51ac536c347dccf9f816f.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:30:11,071 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:30:11,071 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table2 8212c24e55f51ac536c347dccf9f816f 2023-08-10T06:30:11,071 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(888): Instantiated ns2:table2,,1691649009930.8212c24e55f51ac536c347dccf9f816f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:30:11,071 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(7296): checking encryption for 8212c24e55f51ac536c347dccf9f816f 2023-08-10T06:30:11,071 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(7299): checking classloading for 8212c24e55f51ac536c347dccf9f816f 2023-08-10T06:30:11,072 INFO [StoreOpener-8212c24e55f51ac536c347dccf9f816f-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 8212c24e55f51ac536c347dccf9f816f 2023-08-10T06:30:11,074 INFO [StoreOpener-8212c24e55f51ac536c347dccf9f816f-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8212c24e55f51ac536c347dccf9f816f columnFamilyName f 2023-08-10T06:30:11,074 DEBUG [StoreOpener-8212c24e55f51ac536c347dccf9f816f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:30:11,074 INFO [StoreOpener-8212c24e55f51ac536c347dccf9f816f-1 {}] regionserver.HStore(324): Store=8212c24e55f51ac536c347dccf9f816f/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:30:11,075 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/table2/8212c24e55f51ac536c347dccf9f816f 2023-08-10T06:30:11,075 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/table2/8212c24e55f51ac536c347dccf9f816f 2023-08-10T06:30:11,077 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(1076): writing seq id for 8212c24e55f51ac536c347dccf9f816f 2023-08-10T06:30:11,079 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/table2/8212c24e55f51ac536c347dccf9f816f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:30:11,079 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(1093): Opened 8212c24e55f51ac536c347dccf9f816f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11485258400, jitterRate=0.06964804232120514}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-08-10T06:30:11,080 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(992): Region open journal for 8212c24e55f51ac536c347dccf9f816f: 2023-08-10T06:30:11,081 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegionServer(2552): Post open deploy tasks for ns2:table2,,1691649009930.8212c24e55f51ac536c347dccf9f816f., pid=103, masterSystemTime=1691649011063 2023-08-10T06:30:11,082 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegionServer(2579): Finished post open deploy task for ns2:table2,,1691649009930.8212c24e55f51ac536c347dccf9f816f. 2023-08-10T06:30:11,082 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] handler.AssignRegionHandler(158): Opened ns2:table2,,1691649009930.8212c24e55f51ac536c347dccf9f816f. 2023-08-10T06:30:11,082 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=102 updating hbase:meta row=8212c24e55f51ac536c347dccf9f816f, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:30:11,084 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=103, resume processing ppid=102 2023-08-10T06:30:11,084 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=103, ppid=102, state=SUCCESS; OpenRegionProcedure 8212c24e55f51ac536c347dccf9f816f, server=jenkins-hbase3.apache.org,45101,1691648859755 in 171 msec 2023-08-10T06:30:11,086 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=102, resume processing ppid=101 2023-08-10T06:30:11,086 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=102, ppid=101, state=SUCCESS; TransitRegionStateProcedure table=ns2:table2, region=8212c24e55f51ac536c347dccf9f816f, ASSIGN in 328 msec 2023-08-10T06:30:11,087 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=101, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-08-10T06:30:11,087 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649011087"}]},"ts":"1691649011087"} 2023-08-10T06:30:11,088 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:table2, state=ENABLED in hbase:meta 2023-08-10T06:30:11,090 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=101, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_POST_OPERATION 2023-08-10T06:30:11,090 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=101, state=SUCCESS; CreateTableProcedure table=ns2:table2 in 1.1600 sec 2023-08-10T06:30:12,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=101 2023-08-10T06:30:12,041 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:45379/backupUT/backup_1691648993339/ns2/test-16916488687341/archive/data/ns2/test-16916488687341/4897eee4660ec19a1cc212f00aac000c from hbase tables ns2:test-16916488687341 to tables ns2:table2 2023-08-10T06:30:12,041 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceRestoreJob(72): Restore ns2:test-16916488687341 into ns2:table2 2023-08-10T06:30:12,043 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-ns2-table2-1691649012041 from hdfs://localhost:45379/backupUT/backup_1691648993339/ns2/test-16916488687341/archive/data/ns2/test-16916488687341/4897eee4660ec19a1cc212f00aac000c 2023-08-10T06:30:12,049 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@610debcf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:30:12,049 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:30:12,050 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:48482, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:30:12,051 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-08-10T06:30:12,051 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:table2 2023-08-10T06:30:12,054 DEBUG [hconnection-0x10aeecd4-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:30:12,055 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:57286, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:30:12,056 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:table2: 2023-08-10T06:30:12,056 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-08-10T06:30:12,057 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/partitions_dd6eb543-723b-41a7-9a03-bae6af51e212 2023-08-10T06:30:12,063 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:12,063 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:12,064 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:12,064 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:12,553 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:table2' 2023-08-10T06:30:13,032 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/hadoop-3769470778785279656.jar 2023-08-10T06:30:13,033 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:13,033 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:13,034 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:13,034 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:13,034 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:13,035 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:13,035 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:13,035 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:13,036 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:13,036 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:30:13,037 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-08-10T06:30:13,037 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-08-10T06:30:13,038 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-08-10T06:30:13,038 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-08-10T06:30:13,039 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-08-10T06:30:13,039 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-08-10T06:30:13,039 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-08-10T06:30:13,040 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-08-10T06:30:13,040 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-08-10T06:30:13,041 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-08-10T06:30:13,041 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-08-10T06:30:13,041 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:13,042 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:13,042 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:13,042 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:13,042 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:13,043 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:13,043 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:30:13,044 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: ns2:table2 2023-08-10T06:30:13,044 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:30:13,044 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-08-10T06:30:13,044 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-08-10T06:30:13,045 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:30:13,056 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-08-10T06:30:13,132 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691649013100/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-08-10T06:30:13,132 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691649013100/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-08-10T06:30:13,179 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:45379/backupUT/backup_1691648993339/ns2/test-16916488687341/archive/data/ns2/test-16916488687341/4897eee4660ec19a1cc212f00aac000c/f/a272e7ab9ac64bbb812d9b65df841988_SeqId_4_ 2023-08-10T06:30:13,182 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-08-10T06:30:13,184 DEBUG [LocalJobRunner Map Task Executor #0 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a272e7ab9ac64bbb812d9b65df841988_SeqId_4_ 2023-08-10T06:30:13,200 DEBUG [pool-394-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65ddaeab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:30:13,201 DEBUG [hconnection-0x6aef50f1-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:30:13,203 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:57292, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:30:13,204 DEBUG [pool-394-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:30:13,205 DEBUG [pool-394-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-08-10T06:30:13,205 DEBUG [pool-394-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-08-10T06:30:13,211 DEBUG [pool-394-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-ns2-table2-1691649012041/_temporary/0/_temporary/attempt_local1058035449_0016_r_000000_0/f/47bd80e25e7247dbb888815086431593 is 37, key is row10/f:q1/1691648873613/Put/seqid=0 2023-08-10T06:30:13,212 WARN [Thread-1870 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:45101] Specified favored nodes: [jenkins-hbase3.apache.org:45101] 2023-08-10T06:30:14,179 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-ns2-table2-1691649012041 2023-08-10T06:30:14,185 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b8dd3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:30:14,186 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:30:14,188 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:57302, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:30:14,193 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-ns2-table2-1691649012041/_SUCCESS 2023-08-10T06:30:14,196 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:30:14,197 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:48494, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:30:14,205 DEBUG [LoadIncrementalHFiles-0 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 47bd80e25e7247dbb888815086431593 2023-08-10T06:30:14,205 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-ns2-table2-1691649012041/f/47bd80e25e7247dbb888815086431593 first=Optional[row0] last=Optional[row98] 2023-08-10T06:30:14,206 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=ns2:table2,,1691649009930.8212c24e55f51ac536c347dccf9f816f., hostname=jenkins-hbase3.apache.org,45101,1691648859755, seqNum=2 for row with hfile group [{f,hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-ns2-table2-1691649012041/f/47bd80e25e7247dbb888815086431593}] 2023-08-10T06:30:14,211 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-ns2-table2-1691649012041/f/47bd80e25e7247dbb888815086431593 for inclusion in 8212c24e55f51ac536c347dccf9f816f/f 2023-08-10T06:30:14,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-08-10T06:30:14,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HStore(625): Region bounds: first= last= 2023-08-10T06:30:14,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 47bd80e25e7247dbb888815086431593 2023-08-10T06:30:14,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HRegion(2520): Flush status journal for 8212c24e55f51ac536c347dccf9f816f: 2023-08-10T06:30:14,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-ns2-table2-1691649012041/f/47bd80e25e7247dbb888815086431593 to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/staging/jenkins__ns2_table2__ah8h3ual479tvakh0lbujn0i13c9ohbm0i2bkg0o6cfplkp7gglgudoa5srmvpno/f/47bd80e25e7247dbb888815086431593 2023-08-10T06:30:14,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/staging/jenkins__ns2_table2__ah8h3ual479tvakh0lbujn0i13c9ohbm0i2bkg0o6cfplkp7gglgudoa5srmvpno/f/47bd80e25e7247dbb888815086431593 as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/table2/8212c24e55f51ac536c347dccf9f816f/f/2555964ef77c495a9c959b110ef04a47_SeqId_4_ 2023-08-10T06:30:14,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dcdf42b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:30:14,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:30:14,225 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:48498, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:30:14,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:30:14,232 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:57304, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:30:14,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:30:14,245 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/staging/jenkins__ns2_table2__ah8h3ual479tvakh0lbujn0i13c9ohbm0i2bkg0o6cfplkp7gglgudoa5srmvpno/f/47bd80e25e7247dbb888815086431593 into 8212c24e55f51ac536c347dccf9f816f/f as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/table2/8212c24e55f51ac536c347dccf9f816f/f/2555964ef77c495a9c959b110ef04a47_SeqId_4_ - updating store file list. 2023-08-10T06:30:14,250 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/table2/8212c24e55f51ac536c347dccf9f816f/f/2555964ef77c495a9c959b110ef04a47_SeqId_4_ into 8212c24e55f51ac536c347dccf9f816f/f 2023-08-10T06:30:14,250 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/staging/jenkins__ns2_table2__ah8h3ual479tvakh0lbujn0i13c9ohbm0i2bkg0o6cfplkp7gglgudoa5srmvpno/f/47bd80e25e7247dbb888815086431593 into 8212c24e55f51ac536c347dccf9f816f/f (new location: hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/table2/8212c24e55f51ac536c347dccf9f816f/f/2555964ef77c495a9c959b110ef04a47_SeqId_4_) 2023-08-10T06:30:14,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/staging/jenkins__ns2_table2__ah8h3ual479tvakh0lbujn0i13c9ohbm0i2bkg0o6cfplkp7gglgudoa5srmvpno/f/47bd80e25e7247dbb888815086431593 2023-08-10T06:30:14,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53cf030a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:30:14,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:30:14,257 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:48514, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-08-10T06:30:14,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:30:14,264 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:57320, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-08-10T06:30:14,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:30:14,278 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:30:14,278 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-08-10T06:30:14,278 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-08-10T06:30:14,278 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(233): Backup: backup_1691648993339 hdfs://localhost:45379/backupUT/backup_1691648993339/ns2/test-16916488687341/ 2023-08-10T06:30:14,279 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:45379/backupUT/backup_1691648993339/.backup.manifest 2023-08-10T06:30:14,282 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691648993339/.backup.manifest 2023-08-10T06:30:14,282 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(148): Restoring 'ns3:test-16916488687342' to 'ns3:table3_restore' from full backup image hdfs://localhost:45379/backupUT/backup_1691648993339/ns3/test-16916488687342 2023-08-10T06:30:14,288 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] util.RestoreTool(101): Folder tableArchivePath: hdfs://localhost:45379/backupUT/backup_1691648993339/ns3/test-16916488687342/archive/data/ns3/test-16916488687342 does not exists 2023-08-10T06:30:14,288 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] util.RestoreTool(326): find table descriptor but no archive dir for table ns3:test-16916488687342, will only create table 2023-08-10T06:30:14,289 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] util.RestoreTool(487): Creating target table 'ns3:table3_restore' 2023-08-10T06:30:14,290 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'ns3:table3_restore', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:30:14,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=104, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns3:table3_restore 2023-08-10T06:30:14,293 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=104, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_PRE_OPERATION 2023-08-10T06:30:14,294 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns3" qualifier: "table3_restore" procId is: 104 2023-08-10T06:30:14,294 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=104, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-08-10T06:30:14,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=104 2023-08-10T06:30:14,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=104 2023-08-10T06:30:14,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=104 2023-08-10T06:30:14,705 INFO [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 661bd508eacac7c92395a1765adca8df, NAME => 'ns3:table3_restore,,1691649014290.661bd508eacac7c92395a1765adca8df.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:table3_restore', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e 2023-08-10T06:30:14,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=104 2023-08-10T06:30:15,111 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:table3_restore,,1691649014290.661bd508eacac7c92395a1765adca8df.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:30:15,111 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1668): Closing 661bd508eacac7c92395a1765adca8df, disabling compactions & flushes 2023-08-10T06:30:15,111 INFO [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:table3_restore,,1691649014290.661bd508eacac7c92395a1765adca8df. 2023-08-10T06:30:15,111 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:table3_restore,,1691649014290.661bd508eacac7c92395a1765adca8df. 2023-08-10T06:30:15,111 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:table3_restore,,1691649014290.661bd508eacac7c92395a1765adca8df. after waiting 0 ms 2023-08-10T06:30:15,111 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:table3_restore,,1691649014290.661bd508eacac7c92395a1765adca8df. 2023-08-10T06:30:15,111 INFO [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1909): Closed ns3:table3_restore,,1691649014290.661bd508eacac7c92395a1765adca8df. 2023-08-10T06:30:15,111 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1622): Region close journal for 661bd508eacac7c92395a1765adca8df: 2023-08-10T06:30:15,112 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=104, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_ADD_TO_META 2023-08-10T06:30:15,112 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:table3_restore,,1691649014290.661bd508eacac7c92395a1765adca8df.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1691649015112"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691649015112"}]},"ts":"1691649015112"} 2023-08-10T06:30:15,114 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:30:15,114 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=104, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-08-10T06:30:15,114 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649015114"}]},"ts":"1691649015114"} 2023-08-10T06:30:15,115 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:table3_restore, state=ENABLING in hbase:meta 2023-08-10T06:30:15,121 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:table3_restore, region=661bd508eacac7c92395a1765adca8df, ASSIGN}] 2023-08-10T06:30:15,123 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:table3_restore, region=661bd508eacac7c92395a1765adca8df, ASSIGN 2023-08-10T06:30:15,123 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:table3_restore, region=661bd508eacac7c92395a1765adca8df, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45101,1691648859755; forceNewPlan=false, retain=false 2023-08-10T06:30:15,275 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=105 updating hbase:meta row=661bd508eacac7c92395a1765adca8df, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:30:15,277 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE; OpenRegionProcedure 661bd508eacac7c92395a1765adca8df, server=jenkins-hbase3.apache.org,45101,1691648859755}] 2023-08-10T06:30:15,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=104 2023-08-10T06:30:15,431 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] handler.AssignRegionHandler(130): Open ns3:table3_restore,,1691649014290.661bd508eacac7c92395a1765adca8df. 2023-08-10T06:30:15,431 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7254): Opening region: {ENCODED => 661bd508eacac7c92395a1765adca8df, NAME => 'ns3:table3_restore,,1691649014290.661bd508eacac7c92395a1765adca8df.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:30:15,432 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:30:15,432 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table3_restore 661bd508eacac7c92395a1765adca8df 2023-08-10T06:30:15,432 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(888): Instantiated ns3:table3_restore,,1691649014290.661bd508eacac7c92395a1765adca8df.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:30:15,432 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7296): checking encryption for 661bd508eacac7c92395a1765adca8df 2023-08-10T06:30:15,432 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7299): checking classloading for 661bd508eacac7c92395a1765adca8df 2023-08-10T06:30:15,433 INFO [StoreOpener-661bd508eacac7c92395a1765adca8df-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 661bd508eacac7c92395a1765adca8df 2023-08-10T06:30:15,434 INFO [StoreOpener-661bd508eacac7c92395a1765adca8df-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 661bd508eacac7c92395a1765adca8df columnFamilyName f 2023-08-10T06:30:15,434 DEBUG [StoreOpener-661bd508eacac7c92395a1765adca8df-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:30:15,435 INFO [StoreOpener-661bd508eacac7c92395a1765adca8df-1 {}] regionserver.HStore(324): Store=661bd508eacac7c92395a1765adca8df/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:30:15,435 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns3/table3_restore/661bd508eacac7c92395a1765adca8df 2023-08-10T06:30:15,436 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns3/table3_restore/661bd508eacac7c92395a1765adca8df 2023-08-10T06:30:15,438 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1076): writing seq id for 661bd508eacac7c92395a1765adca8df 2023-08-10T06:30:15,440 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns3/table3_restore/661bd508eacac7c92395a1765adca8df/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:30:15,440 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1093): Opened 661bd508eacac7c92395a1765adca8df; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10481714400, jitterRate=-0.023814275860786438}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-08-10T06:30:15,441 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(992): Region open journal for 661bd508eacac7c92395a1765adca8df: 2023-08-10T06:30:15,442 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegionServer(2552): Post open deploy tasks for ns3:table3_restore,,1691649014290.661bd508eacac7c92395a1765adca8df., pid=106, masterSystemTime=1691649015428 2023-08-10T06:30:15,443 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegionServer(2579): Finished post open deploy task for ns3:table3_restore,,1691649014290.661bd508eacac7c92395a1765adca8df. 2023-08-10T06:30:15,443 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] handler.AssignRegionHandler(158): Opened ns3:table3_restore,,1691649014290.661bd508eacac7c92395a1765adca8df. 2023-08-10T06:30:15,443 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=105 updating hbase:meta row=661bd508eacac7c92395a1765adca8df, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:30:15,446 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=106, resume processing ppid=105 2023-08-10T06:30:15,446 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=106, ppid=105, state=SUCCESS; OpenRegionProcedure 661bd508eacac7c92395a1765adca8df, server=jenkins-hbase3.apache.org,45101,1691648859755 in 169 msec 2023-08-10T06:30:15,447 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=105, resume processing ppid=104 2023-08-10T06:30:15,447 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=105, ppid=104, state=SUCCESS; TransitRegionStateProcedure table=ns3:table3_restore, region=661bd508eacac7c92395a1765adca8df, ASSIGN in 327 msec 2023-08-10T06:30:15,447 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=104, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-08-10T06:30:15,448 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649015447"}]},"ts":"1691649015447"} 2023-08-10T06:30:15,449 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:table3_restore, state=ENABLED in hbase:meta 2023-08-10T06:30:15,450 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=104, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_POST_OPERATION 2023-08-10T06:30:15,451 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=104, state=SUCCESS; CreateTableProcedure table=ns3:table3_restore in 1.1600 sec 2023-08-10T06:30:16,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=104 2023-08-10T06:30:16,402 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-08-10T06:30:16,402 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(233): Backup: backup_1691648993339 hdfs://localhost:45379/backupUT/backup_1691648993339/ns3/test-16916488687342/ 2023-08-10T06:30:16,402 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-08-10T06:30:16,404 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster$12(2663): Client=jenkins//172.31.12.81 disable ns2:table2 2023-08-10T06:30:16,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=107, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=ns2:table2 2023-08-10T06:30:16,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=107 2023-08-10T06:30:16,408 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649016407"}]},"ts":"1691649016407"} 2023-08-10T06:30:16,409 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:table2, state=DISABLING in hbase:meta 2023-08-10T06:30:16,411 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(293): Set ns2:table2 to state=DISABLING 2023-08-10T06:30:16,412 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns2:table2, region=8212c24e55f51ac536c347dccf9f816f, UNASSIGN}] 2023-08-10T06:30:16,413 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=108, ppid=107, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns2:table2, region=8212c24e55f51ac536c347dccf9f816f, UNASSIGN 2023-08-10T06:30:16,414 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=108 updating hbase:meta row=8212c24e55f51ac536c347dccf9f816f, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:30:16,415 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE; CloseRegionProcedure 8212c24e55f51ac536c347dccf9f816f, server=jenkins-hbase3.apache.org,45101,1691648859755}] 2023-08-10T06:30:16,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=107 2023-08-10T06:30:16,567 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] handler.UnassignRegionHandler(119): Close 8212c24e55f51ac536c347dccf9f816f 2023-08-10T06:30:16,567 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegion(1668): Closing 8212c24e55f51ac536c347dccf9f816f, disabling compactions & flushes 2023-08-10T06:30:16,567 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegion(1690): Closing region ns2:table2,,1691649009930.8212c24e55f51ac536c347dccf9f816f. 2023-08-10T06:30:16,567 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:table2,,1691649009930.8212c24e55f51ac536c347dccf9f816f. 2023-08-10T06:30:16,567 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegion(1778): Acquired close lock on ns2:table2,,1691649009930.8212c24e55f51ac536c347dccf9f816f. after waiting 0 ms 2023-08-10T06:30:16,567 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegion(1788): Updates disabled for region ns2:table2,,1691649009930.8212c24e55f51ac536c347dccf9f816f. 2023-08-10T06:30:16,568 DEBUG [StoreFileCloser-8212c24e55f51ac536c347dccf9f816f-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 2555964ef77c495a9c959b110ef04a47_SeqId_4_ 2023-08-10T06:30:16,573 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/table2/8212c24e55f51ac536c347dccf9f816f/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2023-08-10T06:30:16,573 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:30:16,574 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegion(1909): Closed ns2:table2,,1691649009930.8212c24e55f51ac536c347dccf9f816f. 2023-08-10T06:30:16,574 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegion(1622): Region close journal for 8212c24e55f51ac536c347dccf9f816f: 2023-08-10T06:30:16,576 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] handler.UnassignRegionHandler(163): Closed 8212c24e55f51ac536c347dccf9f816f 2023-08-10T06:30:16,576 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=108 updating hbase:meta row=8212c24e55f51ac536c347dccf9f816f, regionState=CLOSED 2023-08-10T06:30:16,579 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=109, resume processing ppid=108 2023-08-10T06:30:16,579 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=109, ppid=108, state=SUCCESS; CloseRegionProcedure 8212c24e55f51ac536c347dccf9f816f, server=jenkins-hbase3.apache.org,45101,1691648859755 in 162 msec 2023-08-10T06:30:16,580 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=108, resume processing ppid=107 2023-08-10T06:30:16,580 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=108, ppid=107, state=SUCCESS; TransitRegionStateProcedure table=ns2:table2, region=8212c24e55f51ac536c347dccf9f816f, UNASSIGN in 167 msec 2023-08-10T06:30:16,581 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649016581"}]},"ts":"1691649016581"} 2023-08-10T06:30:16,582 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:table2, state=DISABLED in hbase:meta 2023-08-10T06:30:16,583 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(305): Set ns2:table2 to state=DISABLED 2023-08-10T06:30:16,585 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=107, state=SUCCESS; DisableTableProcedure table=ns2:table2 in 180 msec 2023-08-10T06:30:16,609 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-08-10T06:30:16,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=107 2023-08-10T06:30:16,711 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster$5(2447): Client=jenkins//172.31.12.81 delete ns2:table2 2023-08-10T06:30:16,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=110, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=ns2:table2 2023-08-10T06:30:16,714 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=110, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=ns2:table2 2023-08-10T06:30:16,714 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=110, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=ns2:table2 2023-08-10T06:30:16,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=110 2023-08-10T06:30:16,716 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/table2/8212c24e55f51ac536c347dccf9f816f 2023-08-10T06:30:16,718 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/table2/8212c24e55f51ac536c347dccf9f816f/f, FileablePath, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/table2/8212c24e55f51ac536c347dccf9f816f/recovered.edits] 2023-08-10T06:30:16,722 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/table2/8212c24e55f51ac536c347dccf9f816f/f/2555964ef77c495a9c959b110ef04a47_SeqId_4_ to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/archive/data/ns2/table2/8212c24e55f51ac536c347dccf9f816f/f/2555964ef77c495a9c959b110ef04a47_SeqId_4_ 2023-08-10T06:30:16,724 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/table2/8212c24e55f51ac536c347dccf9f816f/recovered.edits/6.seqid to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/archive/data/ns2/table2/8212c24e55f51ac536c347dccf9f816f/recovered.edits/6.seqid 2023-08-10T06:30:16,725 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/table2/8212c24e55f51ac536c347dccf9f816f 2023-08-10T06:30:16,725 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(293): Archived ns2:table2 regions 2023-08-10T06:30:16,726 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=110, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=ns2:table2 2023-08-10T06:30:16,728 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of ns2:table2 from hbase:meta 2023-08-10T06:30:16,730 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(387): Removing 'ns2:table2' descriptor. 2023-08-10T06:30:16,731 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=110, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=ns2:table2 2023-08-10T06:30:16,731 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(377): Removing 'ns2:table2' from region states. 2023-08-10T06:30:16,731 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns2:table2,,1691649009930.8212c24e55f51ac536c347dccf9f816f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1691649016731"}]},"ts":"9223372036854775807"} 2023-08-10T06:30:16,735 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-08-10T06:30:16,735 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 8212c24e55f51ac536c347dccf9f816f, NAME => 'ns2:table2,,1691649009930.8212c24e55f51ac536c347dccf9f816f.', STARTKEY => '', ENDKEY => ''}] 2023-08-10T06:30:16,735 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(381): Marking 'ns2:table2' as deleted. 2023-08-10T06:30:16,735 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1691649016735"}]},"ts":"9223372036854775807"} 2023-08-10T06:30:16,736 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1678): Deleted table ns2:table2 state from META 2023-08-10T06:30:16,738 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(127): Finished pid=110, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=ns2:table2 2023-08-10T06:30:16,739 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=110, state=SUCCESS; DeleteTableProcedure table=ns2:table2 in 27 msec 2023-08-10T06:30:16,762 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_table3_restore 2023-08-10T06:30:16,762 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns3_table_table3_restore Metrics about Tables on a single HBase RegionServer 2023-08-10T06:30:16,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=110 2023-08-10T06:30:16,816 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster$12(2663): Client=jenkins//172.31.12.81 disable ns3:table3_restore 2023-08-10T06:30:16,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=111, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=ns3:table3_restore 2023-08-10T06:30:16,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=111 2023-08-10T06:30:16,820 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649016820"}]},"ts":"1691649016820"} 2023-08-10T06:30:16,821 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:table3_restore, state=DISABLING in hbase:meta 2023-08-10T06:30:16,822 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(293): Set ns3:table3_restore to state=DISABLING 2023-08-10T06:30:16,823 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns3:table3_restore, region=661bd508eacac7c92395a1765adca8df, UNASSIGN}] 2023-08-10T06:30:16,824 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=112, ppid=111, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns3:table3_restore, region=661bd508eacac7c92395a1765adca8df, UNASSIGN 2023-08-10T06:30:16,825 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=112 updating hbase:meta row=661bd508eacac7c92395a1765adca8df, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:30:16,826 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE; CloseRegionProcedure 661bd508eacac7c92395a1765adca8df, server=jenkins-hbase3.apache.org,45101,1691648859755}] 2023-08-10T06:30:16,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=111 2023-08-10T06:30:16,978 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] handler.UnassignRegionHandler(119): Close 661bd508eacac7c92395a1765adca8df 2023-08-10T06:30:16,979 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] regionserver.HRegion(1668): Closing 661bd508eacac7c92395a1765adca8df, disabling compactions & flushes 2023-08-10T06:30:16,979 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] regionserver.HRegion(1690): Closing region ns3:table3_restore,,1691649014290.661bd508eacac7c92395a1765adca8df. 2023-08-10T06:30:16,979 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:table3_restore,,1691649014290.661bd508eacac7c92395a1765adca8df. 2023-08-10T06:30:16,979 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] regionserver.HRegion(1778): Acquired close lock on ns3:table3_restore,,1691649014290.661bd508eacac7c92395a1765adca8df. after waiting 0 ms 2023-08-10T06:30:16,979 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] regionserver.HRegion(1788): Updates disabled for region ns3:table3_restore,,1691649014290.661bd508eacac7c92395a1765adca8df. 2023-08-10T06:30:16,983 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns3/table3_restore/661bd508eacac7c92395a1765adca8df/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-08-10T06:30:16,984 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:30:16,985 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] regionserver.HRegion(1909): Closed ns3:table3_restore,,1691649014290.661bd508eacac7c92395a1765adca8df. 2023-08-10T06:30:16,985 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] regionserver.HRegion(1622): Region close journal for 661bd508eacac7c92395a1765adca8df: 2023-08-10T06:30:16,986 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] handler.UnassignRegionHandler(163): Closed 661bd508eacac7c92395a1765adca8df 2023-08-10T06:30:16,987 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=112 updating hbase:meta row=661bd508eacac7c92395a1765adca8df, regionState=CLOSED 2023-08-10T06:30:16,989 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=113, resume processing ppid=112 2023-08-10T06:30:16,989 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=113, ppid=112, state=SUCCESS; CloseRegionProcedure 661bd508eacac7c92395a1765adca8df, server=jenkins-hbase3.apache.org,45101,1691648859755 in 162 msec 2023-08-10T06:30:16,991 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=112, resume processing ppid=111 2023-08-10T06:30:16,991 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=112, ppid=111, state=SUCCESS; TransitRegionStateProcedure table=ns3:table3_restore, region=661bd508eacac7c92395a1765adca8df, UNASSIGN in 166 msec 2023-08-10T06:30:16,991 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649016991"}]},"ts":"1691649016991"} 2023-08-10T06:30:16,993 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:table3_restore, state=DISABLED in hbase:meta 2023-08-10T06:30:16,995 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(305): Set ns3:table3_restore to state=DISABLED 2023-08-10T06:30:16,997 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=111, state=SUCCESS; DisableTableProcedure table=ns3:table3_restore in 180 msec 2023-08-10T06:30:17,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=111 2023-08-10T06:30:17,122 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster$5(2447): Client=jenkins//172.31.12.81 delete ns3:table3_restore 2023-08-10T06:30:17,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=114, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=ns3:table3_restore 2023-08-10T06:30:17,125 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=114, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-08-10T06:30:17,125 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=114, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-08-10T06:30:17,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=114 2023-08-10T06:30:17,127 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns3/table3_restore/661bd508eacac7c92395a1765adca8df 2023-08-10T06:30:17,128 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns3/table3_restore/661bd508eacac7c92395a1765adca8df/f, FileablePath, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns3/table3_restore/661bd508eacac7c92395a1765adca8df/recovered.edits] 2023-08-10T06:30:17,132 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns3/table3_restore/661bd508eacac7c92395a1765adca8df/recovered.edits/4.seqid to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/archive/data/ns3/table3_restore/661bd508eacac7c92395a1765adca8df/recovered.edits/4.seqid 2023-08-10T06:30:17,133 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns3/table3_restore/661bd508eacac7c92395a1765adca8df 2023-08-10T06:30:17,133 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(293): Archived ns3:table3_restore regions 2023-08-10T06:30:17,135 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=114, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-08-10T06:30:17,136 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of ns3:table3_restore from hbase:meta 2023-08-10T06:30:17,138 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(387): Removing 'ns3:table3_restore' descriptor. 2023-08-10T06:30:17,139 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=114, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-08-10T06:30:17,139 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(377): Removing 'ns3:table3_restore' from region states. 2023-08-10T06:30:17,139 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns3:table3_restore,,1691649014290.661bd508eacac7c92395a1765adca8df.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1691649017139"}]},"ts":"9223372036854775807"} 2023-08-10T06:30:17,140 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-08-10T06:30:17,140 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 661bd508eacac7c92395a1765adca8df, NAME => 'ns3:table3_restore,,1691649014290.661bd508eacac7c92395a1765adca8df.', STARTKEY => '', ENDKEY => ''}] 2023-08-10T06:30:17,140 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(381): Marking 'ns3:table3_restore' as deleted. 2023-08-10T06:30:17,140 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1691649017140"}]},"ts":"9223372036854775807"} 2023-08-10T06:30:17,141 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1678): Deleted table ns3:table3_restore state from META 2023-08-10T06:30:17,143 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(127): Finished pid=114, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-08-10T06:30:17,144 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=114, state=SUCCESS; DeleteTableProcedure table=ns3:table3_restore in 21 msec 2023-08-10T06:30:17,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=114 2023-08-10T06:30:17,246 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreMultiple Thread=601 (was 597) - Thread LEAK? -, OpenFileDescriptor=675 (was 679), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=133 (was 93) - SystemLoadAverage LEAK? -, ProcessCount=171 (was 169) - ProcessCount LEAK? -, AvailableMemoryMB=1303 (was 1383) 2023-08-10T06:30:17,247 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(130): Thread=601 is superior to 500 2023-08-10T06:30:17,264 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreMultipleOverwriteCommand Thread=601, OpenFileDescriptor=675, MaxFileDescriptor=60000, SystemLoadAverage=133, ProcessCount=171, AvailableMemoryMB=1303 2023-08-10T06:30:17,264 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(130): Thread=601 is superior to 500 2023-08-10T06:30:17,264 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] backup.TestFullRestore(234): create full backup image on multiple tables: command-line 2023-08-10T06:30:17,271 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e92eef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:30:17,274 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:30:17,276 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:57330, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:30:17,278 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:30:17,280 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:48526, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:30:17,289 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-08-10T06:30:17,294 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:30:17,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691649017294 (current time:1691649017294). 2023-08-10T06:30:17,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:30:17,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-08-10T06:30:17,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@321fb150, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:30:17,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:30:17,301 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:57336, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:30:17,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:30:17,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:30:17,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=115, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:30:17,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 115 2023-08-10T06:30:17,306 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:30:17,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=115 2023-08-10T06:30:17,306 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:30:17,308 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:30:17,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=115 2023-08-10T06:30:17,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=115 2023-08-10T06:30:17,713 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:30:17,716 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72}] 2023-08-10T06:30:17,717 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=116, ppid=115, state=RUNNABLE; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:30:17,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45101 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=116 2023-08-10T06:30:17,869 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. 2023-08-10T06:30:17,869 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.HRegion(2819): Flushing b9c493f2c9fbef45de19e8268a534e72 2/2 column families, dataSize=1.50 KB heapSize=2.73 KB 2023-08-10T06:30:17,873 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/meta/cec52c43e8824ebfbbc2cf17b7a9ec5f is 171, key is trslm:hdfs://localhost:45379/backupUT\x00ns2:test-16916488687341/meta:log-roll-map/1691649009064/Put/seqid=0 2023-08-10T06:30:17,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=115 2023-08-10T06:30:18,277 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=717 B at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/meta/cec52c43e8824ebfbbc2cf17b7a9ec5f 2023-08-10T06:30:18,283 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: cec52c43e8824ebfbbc2cf17b7a9ec5f 2023-08-10T06:30:18,285 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/session/75226fb0202e42819224b5e5920a02f2 is 524, key is session:backup_1691648993339/session:context/1691649009893/Put/seqid=0 2023-08-10T06:30:18,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=115 2023-08-10T06:30:18,690 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=820 B at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/session/75226fb0202e42819224b5e5920a02f2 2023-08-10T06:30:18,695 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 75226fb0202e42819224b5e5920a02f2 2023-08-10T06:30:18,696 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/meta/cec52c43e8824ebfbbc2cf17b7a9ec5f as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/cec52c43e8824ebfbbc2cf17b7a9ec5f 2023-08-10T06:30:18,701 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/cec52c43e8824ebfbbc2cf17b7a9ec5f, entries=6, sequenceid=94, filesize=5.8 K 2023-08-10T06:30:18,702 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/session/75226fb0202e42819224b5e5920a02f2 as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/75226fb0202e42819224b5e5920a02f2 2023-08-10T06:30:18,707 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/75226fb0202e42819224b5e5920a02f2, entries=2, sequenceid=94, filesize=5.5 K 2023-08-10T06:30:18,709 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.HRegion(3022): Finished flush of dataSize ~1.50 KB/1537, heapSize ~2.70 KB/2760, currentSize=0 B/0 for b9c493f2c9fbef45de19e8268a534e72 in 839ms, sequenceid=94, compaction requested=true 2023-08-10T06:30:18,709 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.HRegion(2520): Flush status journal for b9c493f2c9fbef45de19e8268a534e72: 2023-08-10T06:30:18,709 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. for snapshot_backup_system completed. 2023-08-10T06:30:18,709 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72.' region-info for snapshot=snapshot_backup_system 2023-08-10T06:30:18,709 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:30:18,709 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/bc10682e7f36491d941e2fe62e2e4100, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/9d9a4feb1c4e44d99cc63e3f68878731, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/f04a99990b2c429eaf1096ef413be3b8, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/9e97dc38b66b476f952990b23fa0a3ca, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/930c578fe3f543e1b6308abfb31c7246, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/24713cefebe0499c812c6d209dea0fd6, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/0ae449898702485faa63081b0f3f028a, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/cec52c43e8824ebfbbc2cf17b7a9ec5f] hfiles 2023-08-10T06:30:18,709 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (1/8): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/bc10682e7f36491d941e2fe62e2e4100 for snapshot=snapshot_backup_system 2023-08-10T06:30:18,709 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (2/8): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/9d9a4feb1c4e44d99cc63e3f68878731 for snapshot=snapshot_backup_system 2023-08-10T06:30:18,709 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (3/8): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/f04a99990b2c429eaf1096ef413be3b8 for snapshot=snapshot_backup_system 2023-08-10T06:30:18,709 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (4/8): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/9e97dc38b66b476f952990b23fa0a3ca for snapshot=snapshot_backup_system 2023-08-10T06:30:18,709 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (5/8): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/930c578fe3f543e1b6308abfb31c7246 for snapshot=snapshot_backup_system 2023-08-10T06:30:18,709 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (6/8): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/24713cefebe0499c812c6d209dea0fd6 for snapshot=snapshot_backup_system 2023-08-10T06:30:18,709 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (7/8): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/0ae449898702485faa63081b0f3f028a for snapshot=snapshot_backup_system 2023-08-10T06:30:18,709 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (8/8): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/cec52c43e8824ebfbbc2cf17b7a9ec5f for snapshot=snapshot_backup_system 2023-08-10T06:30:18,710 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/e88678d5b4b14eecabac817daf12539e, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/37561b9a8ae2406bb6d053c33aeff31c, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/883911c19cab473f8bf801f934c90bc1, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/0c7705a45c554d6a97fa37fac319ceff, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/398a15bfffb84acfadb1be252c2e133a, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/adceb6320b02451abd02beed6c69c000, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/68fd0cf6c3094cc7af1a5176829fd9a4, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/40f9fbdeac5b49dbb654e429c177708d, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/75226fb0202e42819224b5e5920a02f2] hfiles 2023-08-10T06:30:18,710 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (1/9): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/e88678d5b4b14eecabac817daf12539e for snapshot=snapshot_backup_system 2023-08-10T06:30:18,710 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (2/9): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/37561b9a8ae2406bb6d053c33aeff31c for snapshot=snapshot_backup_system 2023-08-10T06:30:18,710 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (3/9): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/883911c19cab473f8bf801f934c90bc1 for snapshot=snapshot_backup_system 2023-08-10T06:30:18,710 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (4/9): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/0c7705a45c554d6a97fa37fac319ceff for snapshot=snapshot_backup_system 2023-08-10T06:30:18,710 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (5/9): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/398a15bfffb84acfadb1be252c2e133a for snapshot=snapshot_backup_system 2023-08-10T06:30:18,710 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (6/9): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/adceb6320b02451abd02beed6c69c000 for snapshot=snapshot_backup_system 2023-08-10T06:30:18,710 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (7/9): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/68fd0cf6c3094cc7af1a5176829fd9a4 for snapshot=snapshot_backup_system 2023-08-10T06:30:18,710 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (8/9): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/40f9fbdeac5b49dbb654e429c177708d for snapshot=snapshot_backup_system 2023-08-10T06:30:18,710 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (9/9): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/75226fb0202e42819224b5e5920a02f2 for snapshot=snapshot_backup_system 2023-08-10T06:30:18,722 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. 2023-08-10T06:30:18,722 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=116 2023-08-10T06:30:18,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster(3994): Remote procedure done, pid=116 2023-08-10T06:30:18,722 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:30:18,724 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=116, ppid=115, state=RUNNABLE; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:30:18,727 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=116, resume processing ppid=115 2023-08-10T06:30:18,727 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:30:18,727 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=116, ppid=115, state=SUCCESS; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72 in 1.0110 sec 2023-08-10T06:30:18,728 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:30:18,728 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:30:18,729 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-08-10T06:30:18,729 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_backup_system 2023-08-10T06:30:19,150 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:30:19,164 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:30:19,165 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_backup_system 2023-08-10T06:30:19,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=115 2023-08-10T06:30:19,583 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:30:19,583 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 115 2023-08-10T06:30:19,585 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=115, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2810 sec 2023-08-10T06:30:21,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=115 2023-08-10T06:30:21,413 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(115): Backup backup_1691649017271 started at 1691649021413. 2023-08-10T06:30:21,415 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(119): Backup session backup_1691649017271 has been started. 2023-08-10T06:30:21,421 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-08-10T06:30:21,421 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-08-10T06:30:21,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-08-10T06:30:21,426 INFO [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-08-10T06:30:21,426 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-08-10T06:30:21,427 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-08-10T06:30:21,427 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-08-10T06:30:21,428 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:30:21,430 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:30:21,430 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:30:21,430 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:30:21,430 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-08-10T06:30:21,431 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-08-10T06:30:21,431 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-08-10T06:30:21,431 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:30:21,431 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-08-10T06:30:21,436 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-08-10T06:30:21,436 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-08-10T06:30:21,436 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-08-10T06:30:21,436 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-08-10T06:30:21,437 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-08-10T06:30:21,438 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-08-10T06:30:21,438 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,45101,1691648859755' joining acquired barrier for procedure (rolllog) in zk 2023-08-10T06:30:21,440 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-08-10T06:30:21,440 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:30:21,440 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:30:21,440 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:30:21,440 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-08-10T06:30:21,440 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:30:21,440 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:30:21,441 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:30:21,441 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:30:21,441 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:30:21,441 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,45101,1691648859755' joining acquired barrier for procedure 'rolllog' on coordinator 2023-08-10T06:30:21,441 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@4ff4195[Count = 0] remaining members to acquire global barrier 2023-08-10T06:30:21,441 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-08-10T06:30:21,441 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-08-10T06:30:21,443 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-08-10T06:30:21,443 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-08-10T06:30:21,443 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-08-10T06:30:21,443 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-08-10T06:30:21,447 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-08-10T06:30:21,448 DEBUG [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:30:21,448 INFO [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1691648997529 highest: 1691648997529 on jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:30:21,448 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta:.meta(num 1691648997511) roll requested 2023-08-10T06:30:21,465 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691649021448.meta, exclude list is [], retry=0 2023-08-10T06:30:21,473 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK] 2023-08-10T06:30:21,478 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691648997511.meta with entries=24, filesize=5.75 KB; new WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691649021448.meta 2023-08-10T06:30:21,478 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK]] 2023-08-10T06:30:21,478 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691648997511.meta is not closed yet, will try archiving it next time 2023-08-10T06:30:21,478 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C45101%2C1691648859755:(num 1691648997529) roll requested 2023-08-10T06:30:21,494 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691649021478, exclude list is [], retry=0 2023-08-10T06:30:21,501 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK] 2023-08-10T06:30:21,502 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691648997529 with entries=15, filesize=4.08 KB; new WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691649021478 2023-08-10T06:30:21,503 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK]] 2023-08-10T06:30:21,503 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691648997529 is not closed yet, will try archiving it next time 2023-08-10T06:30:21,503 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691648980808 to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/oldWALs/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691648980808 2023-08-10T06:30:21,506 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691648997529 not finished, retry = 0 2023-08-10T06:30:21,508 DEBUG [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 60 2023-08-10T06:30:21,509 INFO [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1691649021478 on jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:30:21,517 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-08-10T06:30:21,517 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,45101,1691648859755' in zk 2023-08-10T06:30:21,519 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-08-10T06:30:21,519 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:30:21,519 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:30:21,519 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:30:21,519 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-08-10T06:30:21,519 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:30:21,520 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:30:21,521 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:30:21,521 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:30:21,521 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:30:21,522 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:30:21,522 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:30:21,522 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:30:21,523 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,45101,1691648859755': 2023-08-10T06:30:21,523 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,45101,1691648859755' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-08-10T06:30:21,523 INFO [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-08-10T06:30:21,523 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-08-10T06:30:21,523 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-08-10T06:30:21,523 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-08-10T06:30:21,523 INFO [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-08-10T06:30:21,525 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-08-10T06:30:21,526 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-08-10T06:30:21,526 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:30:21,526 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:30:21,526 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:30:21,526 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:30:21,526 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:30:21,526 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:30:21,526 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:30:21,526 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:30:21,527 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:30:21,527 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:30:21,527 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:30:21,528 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:30:21,528 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:30:21,528 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:30:21,534 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:30:21,534 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:30:21,534 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:30:21,534 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-08-10T06:30:21,535 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-08-10T06:30:21,535 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:30:21,535 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:30:21,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-08-10T06:30:21,639 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1691649021638_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } 2023-08-10T06:30:21,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691649021639 (current time:1691649021639). 2023-08-10T06:30:21,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:30:21,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1691649021638_ns2_test-16916488687341 VERSION not specified, setting to 2 2023-08-10T06:30:21,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53529563, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:30:21,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:30:21,649 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:54388, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:30:21,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:30:21,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:30:21,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=117, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1691649021638_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } 2023-08-10T06:30:21,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1691649021638_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 }, snapshot procedure id = 117 2023-08-10T06:30:21,652 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1691649021638_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:30:21,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=117 2023-08-10T06:30:21,653 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1691649021638_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:30:21,655 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1691649021638_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:30:21,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=117 2023-08-10T06:30:21,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=117 2023-08-10T06:30:22,062 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1691649021638_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:30:22,063 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE; SnapshotRegionProcedure 4897eee4660ec19a1cc212f00aac000c}] 2023-08-10T06:30:22,064 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=118, ppid=117, state=RUNNABLE; SnapshotRegionProcedure 4897eee4660ec19a1cc212f00aac000c 2023-08-10T06:30:22,075 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-08-10T06:30:22,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45101 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=118 2023-08-10T06:30:22,216 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-16916488687341,,1691648901068.4897eee4660ec19a1cc212f00aac000c. 2023-08-10T06:30:22,216 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] regionserver.HRegion(2520): Flush status journal for 4897eee4660ec19a1cc212f00aac000c: 2023-08-10T06:30:22,216 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-16916488687341,,1691648901068.4897eee4660ec19a1cc212f00aac000c. for snapshot_1691649021638_ns2_test-16916488687341 completed. 2023-08-10T06:30:22,216 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] snapshot.SnapshotManifest(240): Storing 'ns2:test-16916488687341,,1691648901068.4897eee4660ec19a1cc212f00aac000c.' region-info for snapshot=snapshot_1691649021638_ns2_test-16916488687341 2023-08-10T06:30:22,216 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:30:22,216 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/4897eee4660ec19a1cc212f00aac000c/f/a272e7ab9ac64bbb812d9b65df841988_SeqId_4_] hfiles 2023-08-10T06:30:22,216 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/4897eee4660ec19a1cc212f00aac000c/f/a272e7ab9ac64bbb812d9b65df841988_SeqId_4_ for snapshot=snapshot_1691649021638_ns2_test-16916488687341 2023-08-10T06:30:22,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=117 2023-08-10T06:30:22,622 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-16916488687341,,1691648901068.4897eee4660ec19a1cc212f00aac000c. 2023-08-10T06:30:22,622 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=118 2023-08-10T06:30:22,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster(3994): Remote procedure done, pid=118 2023-08-10T06:30:22,623 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1691649021638_ns2_test-16916488687341 on region 4897eee4660ec19a1cc212f00aac000c 2023-08-10T06:30:22,624 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=118, ppid=117, state=RUNNABLE; SnapshotRegionProcedure 4897eee4660ec19a1cc212f00aac000c 2023-08-10T06:30:22,626 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=118, resume processing ppid=117 2023-08-10T06:30:22,626 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1691649021638_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:30:22,626 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=118, ppid=117, state=SUCCESS; SnapshotRegionProcedure 4897eee4660ec19a1cc212f00aac000c in 563 msec 2023-08-10T06:30:22,627 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1691649021638_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:30:22,627 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1691649021638_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:30:22,627 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1691649021638_ns2_test-16916488687341 2023-08-10T06:30:22,628 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_1691649021638_ns2_test-16916488687341 2023-08-10T06:30:22,637 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1691649021638_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:30:22,641 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1691649021638_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:30:22,642 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_1691649021638_ns2_test-16916488687341 to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691649021638_ns2_test-16916488687341 2023-08-10T06:30:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=117 2023-08-10T06:30:23,457 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1691649021638_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:30:23,457 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1691649021638_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 }, snapshot procedure id = 117 2023-08-10T06:30:23,459 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=117, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1691649021638_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } in 1.8070 sec 2023-08-10T06:30:23,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=117 2023-08-10T06:30:23,760 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1691649023760_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } 2023-08-10T06:30:23,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691649023760 (current time:1691649023760). 2023-08-10T06:30:23,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:30:23,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1691649023760_ns3_test-16916488687342 VERSION not specified, setting to 2 2023-08-10T06:30:23,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b48fd61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:30:23,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:30:23,768 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:54404, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:30:23,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:30:23,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:30:23,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=119, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1691649023760_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } 2023-08-10T06:30:23,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1691649023760_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 }, snapshot procedure id = 119 2023-08-10T06:30:23,772 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1691649023760_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:30:23,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=119 2023-08-10T06:30:23,772 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1691649023760_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:30:23,774 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1691649023760_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:30:23,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=119 2023-08-10T06:30:24,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=119 2023-08-10T06:30:24,181 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1691649023760_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:30:24,181 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE; SnapshotRegionProcedure 77e10d7389ad2c7a8dfcbed4a1e8d4af}] 2023-08-10T06:30:24,185 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=120, ppid=119, state=RUNNABLE; SnapshotRegionProcedure 77e10d7389ad2c7a8dfcbed4a1e8d4af 2023-08-10T06:30:24,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45101 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=120 2023-08-10T06:30:24,337 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=120}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-16916488687342,,1691648905436.77e10d7389ad2c7a8dfcbed4a1e8d4af. 2023-08-10T06:30:24,337 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=120}] regionserver.HRegion(2520): Flush status journal for 77e10d7389ad2c7a8dfcbed4a1e8d4af: 2023-08-10T06:30:24,337 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=120}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-16916488687342,,1691648905436.77e10d7389ad2c7a8dfcbed4a1e8d4af. for snapshot_1691649023760_ns3_test-16916488687342 completed. 2023-08-10T06:30:24,337 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=120}] snapshot.SnapshotManifest(240): Storing 'ns3:test-16916488687342,,1691648905436.77e10d7389ad2c7a8dfcbed4a1e8d4af.' region-info for snapshot=snapshot_1691649023760_ns3_test-16916488687342 2023-08-10T06:30:24,337 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=120}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:30:24,337 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=120}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-08-10T06:30:24,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=119 2023-08-10T06:30:24,742 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=120}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-16916488687342,,1691648905436.77e10d7389ad2c7a8dfcbed4a1e8d4af. 2023-08-10T06:30:24,742 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=120 2023-08-10T06:30:24,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster(3994): Remote procedure done, pid=120 2023-08-10T06:30:24,743 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1691649023760_ns3_test-16916488687342 on region 77e10d7389ad2c7a8dfcbed4a1e8d4af 2023-08-10T06:30:24,745 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=120, ppid=119, state=RUNNABLE; SnapshotRegionProcedure 77e10d7389ad2c7a8dfcbed4a1e8d4af 2023-08-10T06:30:24,747 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=120, resume processing ppid=119 2023-08-10T06:30:24,747 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1691649023760_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:30:24,747 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=120, ppid=119, state=SUCCESS; SnapshotRegionProcedure 77e10d7389ad2c7a8dfcbed4a1e8d4af in 565 msec 2023-08-10T06:30:24,748 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1691649023760_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:30:24,748 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1691649023760_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:30:24,748 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1691649023760_ns3_test-16916488687342 2023-08-10T06:30:24,749 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_1691649023760_ns3_test-16916488687342 2023-08-10T06:30:24,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=119 2023-08-10T06:30:25,158 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1691649023760_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:30:25,163 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1691649023760_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:30:25,163 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_1691649023760_ns3_test-16916488687342 to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691649023760_ns3_test-16916488687342 2023-08-10T06:30:25,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=119 2023-08-10T06:30:25,981 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1691649023760_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:30:25,981 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1691649023760_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 }, snapshot procedure id = 119 2023-08-10T06:30:25,982 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=119, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1691649023760_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } in 2.2120 sec 2023-08-10T06:30:26,762 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_table3_restore 2023-08-10T06:30:27,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=119 2023-08-10T06:30:27,879 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1691649017271 2023-08-10T06:30:27,879 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-08-10T06:30:27,879 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2023-08-10T06:30:27,879 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1691649017271_ns3:test-16916488687342 2023-08-10T06:30:27,879 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1691649023760_ns3_test-16916488687342 to hdfs://localhost:45379/backupUT/backup_1691649017271/ns3/test-16916488687342/ 2023-08-10T06:30:27,879 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-08-10T06:30:27,902 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:45379, inputRoot=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e 2023-08-10T06:30:27,902 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,33705,1691648856595_-854900287_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:45379/backupUT/backup_1691649017271/ns3/test-16916488687342, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:45379/backupUT/backup_1691649017271/ns3/test-16916488687342/.hbase-snapshot/.tmp/snapshot_1691649023760_ns3_test-16916488687342 2023-08-10T06:30:27,902 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:45379, inputRoot=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e, snapshotDir=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691649023760_ns3_test-16916488687342. 2023-08-10T06:30:27,908 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691649023760_ns3_test-16916488687342 to hdfs://localhost:45379/backupUT/backup_1691649017271/ns3/test-16916488687342/.hbase-snapshot/.tmp/snapshot_1691649023760_ns3_test-16916488687342 2023-08-10T06:30:28,326 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:28,326 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:28,327 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:28,327 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:29,368 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/hadoop-8474330551751923120.jar 2023-08-10T06:30:29,369 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:29,369 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:29,369 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:29,370 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:29,370 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:29,371 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:29,371 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:29,371 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:29,372 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:29,373 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:30:29,373 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-08-10T06:30:29,374 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-08-10T06:30:29,374 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-08-10T06:30:29,375 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-08-10T06:30:29,375 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-08-10T06:30:29,375 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-08-10T06:30:29,376 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-08-10T06:30:29,376 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-08-10T06:30:29,377 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-08-10T06:30:29,377 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-08-10T06:30:29,378 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-08-10T06:30:29,378 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:30:29,379 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:30:29,380 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:30:29,380 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:30:29,380 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:30:29,381 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:30:29,381 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:30:29,426 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1691649023760_ns3_test-16916488687342' hfile list 2023-08-10T06:30:29,500 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691649029464/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-08-10T06:30:29,500 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691649029464/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-08-10T06:30:30,528 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-08-10T06:30:30,529 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-08-10T06:30:30,535 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1691649023760_ns3_test-16916488687342 2023-08-10T06:30:30,535 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1691649023760_ns3_test-16916488687342 finished. 2023-08-10T06:30:30,535 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1691649017271_ns2:test-16916488687341 2023-08-10T06:30:30,535 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1691649021638_ns2_test-16916488687341 to hdfs://localhost:45379/backupUT/backup_1691649017271/ns2/test-16916488687341/ 2023-08-10T06:30:30,535 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-08-10T06:30:30,557 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:45379, inputRoot=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e 2023-08-10T06:30:30,557 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,33705,1691648856595_-2021985341_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:45379/backupUT/backup_1691649017271/ns2/test-16916488687341, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:45379/backupUT/backup_1691649017271/ns2/test-16916488687341/.hbase-snapshot/.tmp/snapshot_1691649021638_ns2_test-16916488687341 2023-08-10T06:30:30,557 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:45379, inputRoot=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e, snapshotDir=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691649021638_ns2_test-16916488687341. 2023-08-10T06:30:30,564 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691649021638_ns2_test-16916488687341 to hdfs://localhost:45379/backupUT/backup_1691649017271/ns2/test-16916488687341/.hbase-snapshot/.tmp/snapshot_1691649021638_ns2_test-16916488687341 2023-08-10T06:30:30,978 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:30,978 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:30,979 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:30,979 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:31,991 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/hadoop-7137682580524639151.jar 2023-08-10T06:30:31,991 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:31,991 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:31,992 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:31,992 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:31,992 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:31,993 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:31,993 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:31,993 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:31,994 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:31,994 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:30:31,995 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-08-10T06:30:31,995 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-08-10T06:30:31,996 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-08-10T06:30:31,996 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-08-10T06:30:31,996 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-08-10T06:30:31,997 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-08-10T06:30:31,997 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-08-10T06:30:31,997 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-08-10T06:30:31,998 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-08-10T06:30:31,998 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-08-10T06:30:31,998 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-08-10T06:30:31,999 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:30:31,999 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:30:32,000 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:30:32,000 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:30:32,000 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:30:32,001 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:30:32,002 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:30:32,044 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1691649021638_ns2_test-16916488687341' hfile list 2023-08-10T06:30:32,046 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-08-10T06:30:32,120 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691649032084/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-08-10T06:30:32,120 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691649032084/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-08-10T06:30:32,174 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-08-10T06:30:32,200 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/4897eee4660ec19a1cc212f00aac000c/f/a272e7ab9ac64bbb812d9b65df841988_SeqId_4_ output=hdfs://localhost:45379/backupUT/backup_1691649017271/ns2/test-16916488687341/archive/data/ns2/test-16916488687341/4897eee4660ec19a1cc212f00aac000c/f/a272e7ab9ac64bbb812d9b65df841988_SeqId_4_ 2023-08-10T06:30:32,201 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8725 (8.5 K) time=0sec 8.321M/sec 2023-08-10T06:30:32,265 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-08-10T06:30:33,159 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-08-10T06:30:33,160 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-08-10T06:30:33,169 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1691649021638_ns2_test-16916488687341 2023-08-10T06:30:33,169 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1691649021638_ns2_test-16916488687341 finished. 2023-08-10T06:30:33,169 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(1014): ns3:test-16916488687342 2023-08-10T06:30:33,169 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(1014): ns2:test-16916488687341 2023-08-10T06:30:33,182 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691649017271 2023-08-10T06:30:33,182 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-08-10T06:30:33,588 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:45379/backupUT/backup_1691649017271/.backup.manifest 2023-08-10T06:30:33,588 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691649017271 2023-08-10T06:30:33,588 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-08-10T06:30:33,996 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:45379/backupUT/backup_1691649017271/.backup.manifest 2023-08-10T06:30:33,996 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(358): Backup backup_1691649017271 finished: type=FULL,tablelist=ns3:test-16916488687342;ns2:test-16916488687341,targetRootDir=hdfs://localhost:45379/backupUT,startts=1691649021413,completets=1691649033182,bytescopied=0 2023-08-10T06:30:33,996 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-08-10T06:30:33,996 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1691649023760_ns3_test-16916488687342 2023-08-10T06:30:33,997 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1691649023760_ns3_test-16916488687342" 2023-08-10T06:30:33,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1691649023760_ns3_test-16916488687342 2023-08-10T06:30:34,000 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1691649023760_ns3_test-16916488687342 for backup backup_1691649017271 succeeded. 2023-08-10T06:30:34,000 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1691649021638_ns2_test-16916488687341 2023-08-10T06:30:34,001 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1691649021638_ns2_test-16916488687341" 2023-08-10T06:30:34,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1691649021638_ns2_test-16916488687341 2023-08-10T06:30:34,003 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1691649021638_ns2_test-16916488687341 for backup backup_1691649017271 succeeded. 2023-08-10T06:30:34,004 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-08-10T06:30:34,008 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-08-10T06:30:34,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-08-10T06:30:34,010 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-08-10T06:30:34,011 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-08-10T06:30:34,017 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(376): Backup backup_1691649017271 completed. 2023-08-10T06:30:34,017 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:30:34,027 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] backup.RestoreDriver(101): Found -overwrite option in restore command, will overwrite to existing table if any in the restore target 2023-08-10T06:30:34,033 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14909199, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:30:34,040 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:30:34,041 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:55188, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:30:34,045 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:45379/backupUT/backup_1691649017271/.backup.manifest 2023-08-10T06:30:34,046 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649017271/.backup.manifest 2023-08-10T06:30:34,047 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:45379/backupUT/backup_1691649017271/.backup.manifest 2023-08-10T06:30:34,048 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649017271/.backup.manifest 2023-08-10T06:30:34,049 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:45379/backupUT/backup_1691649017271/.backup.manifest 2023-08-10T06:30:34,050 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649017271/.backup.manifest 2023-08-10T06:30:34,050 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(148): Restoring 'ns2:test-16916488687341' to 'ns2:test-16916488687341' from full backup image hdfs://localhost:45379/backupUT/backup_1691649017271/ns2/test-16916488687341 2023-08-10T06:30:34,057 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] util.RestoreTool(476): Truncating exising target table 'ns2:test-16916488687341', preserving region splits 2023-08-10T06:30:34,058 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:30:34,061 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:49266, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:30:34,062 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster$12(2663): Client=jenkins//172.31.12.81 disable ns2:test-16916488687341 2023-08-10T06:30:34,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=121, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=ns2:test-16916488687341 2023-08-10T06:30:34,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=121 2023-08-10T06:30:34,066 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16916488687341","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649034065"}]},"ts":"1691649034065"} 2023-08-10T06:30:34,067 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16916488687341, state=DISABLING in hbase:meta 2023-08-10T06:30:34,068 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(293): Set ns2:test-16916488687341 to state=DISABLING 2023-08-10T06:30:34,069 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns2:test-16916488687341, region=4897eee4660ec19a1cc212f00aac000c, UNASSIGN}] 2023-08-10T06:30:34,070 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=122, ppid=121, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns2:test-16916488687341, region=4897eee4660ec19a1cc212f00aac000c, UNASSIGN 2023-08-10T06:30:34,070 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=122 updating hbase:meta row=4897eee4660ec19a1cc212f00aac000c, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:30:34,071 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=123, ppid=122, state=RUNNABLE; CloseRegionProcedure 4897eee4660ec19a1cc212f00aac000c, server=jenkins-hbase3.apache.org,45101,1691648859755}] 2023-08-10T06:30:34,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=121 2023-08-10T06:30:34,223 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] handler.UnassignRegionHandler(119): Close 4897eee4660ec19a1cc212f00aac000c 2023-08-10T06:30:34,223 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(1668): Closing 4897eee4660ec19a1cc212f00aac000c, disabling compactions & flushes 2023-08-10T06:30:34,223 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(1690): Closing region ns2:test-16916488687341,,1691648901068.4897eee4660ec19a1cc212f00aac000c. 2023-08-10T06:30:34,223 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16916488687341,,1691648901068.4897eee4660ec19a1cc212f00aac000c. 2023-08-10T06:30:34,223 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16916488687341,,1691648901068.4897eee4660ec19a1cc212f00aac000c. after waiting 0 ms 2023-08-10T06:30:34,223 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16916488687341,,1691648901068.4897eee4660ec19a1cc212f00aac000c. 2023-08-10T06:30:34,225 DEBUG [StoreFileCloser-4897eee4660ec19a1cc212f00aac000c-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a272e7ab9ac64bbb812d9b65df841988_SeqId_4_ 2023-08-10T06:30:34,230 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/4897eee4660ec19a1cc212f00aac000c/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2023-08-10T06:30:34,230 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:30:34,231 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(1909): Closed ns2:test-16916488687341,,1691648901068.4897eee4660ec19a1cc212f00aac000c. 2023-08-10T06:30:34,231 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(1622): Region close journal for 4897eee4660ec19a1cc212f00aac000c: 2023-08-10T06:30:34,233 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] handler.UnassignRegionHandler(163): Closed 4897eee4660ec19a1cc212f00aac000c 2023-08-10T06:30:34,233 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=122 updating hbase:meta row=4897eee4660ec19a1cc212f00aac000c, regionState=CLOSED 2023-08-10T06:30:34,236 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=123, resume processing ppid=122 2023-08-10T06:30:34,236 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=123, ppid=122, state=SUCCESS; CloseRegionProcedure 4897eee4660ec19a1cc212f00aac000c, server=jenkins-hbase3.apache.org,45101,1691648859755 in 163 msec 2023-08-10T06:30:34,237 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=122, resume processing ppid=121 2023-08-10T06:30:34,237 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=122, ppid=121, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-16916488687341, region=4897eee4660ec19a1cc212f00aac000c, UNASSIGN in 167 msec 2023-08-10T06:30:34,238 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16916488687341","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649034238"}]},"ts":"1691649034238"} 2023-08-10T06:30:34,239 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16916488687341, state=DISABLED in hbase:meta 2023-08-10T06:30:34,241 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(305): Set ns2:test-16916488687341 to state=DISABLED 2023-08-10T06:30:34,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=121, state=SUCCESS; DisableTableProcedure table=ns2:test-16916488687341 in 179 msec 2023-08-10T06:30:34,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=121 2023-08-10T06:30:34,369 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster$6(2479): Client=jenkins//172.31.12.81 truncate ns2:test-16916488687341 2023-08-10T06:30:34,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=124, state=RUNNABLE:TRUNCATE_TABLE_PRE_OPERATION; TruncateTableProcedure (table=ns2:test-16916488687341 preserveSplits=true) 2023-08-10T06:30:34,372 DEBUG [PEWorker-2 {}] procedure.TruncateTableProcedure(87): waiting for 'ns2:test-16916488687341' regions in transition 2023-08-10T06:30:34,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=124 2023-08-10T06:30:34,374 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/4897eee4660ec19a1cc212f00aac000c 2023-08-10T06:30:34,376 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/4897eee4660ec19a1cc212f00aac000c/f, FileablePath, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/4897eee4660ec19a1cc212f00aac000c/recovered.edits] 2023-08-10T06:30:34,381 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/4897eee4660ec19a1cc212f00aac000c/f/a272e7ab9ac64bbb812d9b65df841988_SeqId_4_ to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/archive/data/ns2/test-16916488687341/4897eee4660ec19a1cc212f00aac000c/f/a272e7ab9ac64bbb812d9b65df841988_SeqId_4_ 2023-08-10T06:30:34,383 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/4897eee4660ec19a1cc212f00aac000c/recovered.edits/9.seqid to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/archive/data/ns2/test-16916488687341/4897eee4660ec19a1cc212f00aac000c/recovered.edits/9.seqid 2023-08-10T06:30:34,384 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/4897eee4660ec19a1cc212f00aac000c 2023-08-10T06:30:34,384 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(293): Archived ns2:test-16916488687341 regions 2023-08-10T06:30:34,388 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of ns2:test-16916488687341 from hbase:meta 2023-08-10T06:30:34,390 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(387): Removing 'ns2:test-16916488687341' descriptor. 2023-08-10T06:30:34,391 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(377): Removing 'ns2:test-16916488687341' from region states. 2023-08-10T06:30:34,391 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns2:test-16916488687341,,1691648901068.4897eee4660ec19a1cc212f00aac000c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1691649034391"}]},"ts":"9223372036854775807"} 2023-08-10T06:30:34,393 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-08-10T06:30:34,393 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 4897eee4660ec19a1cc212f00aac000c, NAME => 'ns2:test-16916488687341,,1691648901068.4897eee4660ec19a1cc212f00aac000c.', STARTKEY => '', ENDKEY => ''}] 2023-08-10T06:30:34,393 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(381): Marking 'ns2:test-16916488687341' as deleted. 2023-08-10T06:30:34,393 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns2:test-16916488687341","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1691649034393"}]},"ts":"9223372036854775807"} 2023-08-10T06:30:34,394 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1678): Deleted table ns2:test-16916488687341 state from META 2023-08-10T06:30:34,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=124 2023-08-10T06:30:34,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=124 2023-08-10T06:30:34,805 INFO [RegionOpenAndInit-ns2:test-16916488687341-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 25a1c5b32e608be67dca00df059e0053, NAME => 'ns2:test-16916488687341,,1691649034386.25a1c5b32e608be67dca00df059e0053.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-16916488687341', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e 2023-08-10T06:30:34,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=124 2023-08-10T06:30:35,212 DEBUG [RegionOpenAndInit-ns2:test-16916488687341-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:test-16916488687341,,1691649034386.25a1c5b32e608be67dca00df059e0053.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:30:35,212 DEBUG [RegionOpenAndInit-ns2:test-16916488687341-pool-0 {}] regionserver.HRegion(1668): Closing 25a1c5b32e608be67dca00df059e0053, disabling compactions & flushes 2023-08-10T06:30:35,212 INFO [RegionOpenAndInit-ns2:test-16916488687341-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:test-16916488687341,,1691649034386.25a1c5b32e608be67dca00df059e0053. 2023-08-10T06:30:35,212 DEBUG [RegionOpenAndInit-ns2:test-16916488687341-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16916488687341,,1691649034386.25a1c5b32e608be67dca00df059e0053. 2023-08-10T06:30:35,212 DEBUG [RegionOpenAndInit-ns2:test-16916488687341-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16916488687341,,1691649034386.25a1c5b32e608be67dca00df059e0053. after waiting 0 ms 2023-08-10T06:30:35,212 DEBUG [RegionOpenAndInit-ns2:test-16916488687341-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16916488687341,,1691649034386.25a1c5b32e608be67dca00df059e0053. 2023-08-10T06:30:35,212 INFO [RegionOpenAndInit-ns2:test-16916488687341-pool-0 {}] regionserver.HRegion(1909): Closed ns2:test-16916488687341,,1691649034386.25a1c5b32e608be67dca00df059e0053. 2023-08-10T06:30:35,212 DEBUG [RegionOpenAndInit-ns2:test-16916488687341-pool-0 {}] regionserver.HRegion(1622): Region close journal for 25a1c5b32e608be67dca00df059e0053: 2023-08-10T06:30:35,213 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:test-16916488687341,,1691649034386.25a1c5b32e608be67dca00df059e0053.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1691649035213"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691649035213"}]},"ts":"1691649035213"} 2023-08-10T06:30:35,214 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:30:35,215 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16916488687341","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649035215"}]},"ts":"1691649035215"} 2023-08-10T06:30:35,216 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16916488687341, state=ENABLING in hbase:meta 2023-08-10T06:30:35,220 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16916488687341, region=25a1c5b32e608be67dca00df059e0053, ASSIGN}] 2023-08-10T06:30:35,222 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16916488687341, region=25a1c5b32e608be67dca00df059e0053, ASSIGN 2023-08-10T06:30:35,222 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:test-16916488687341, region=25a1c5b32e608be67dca00df059e0053, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45101,1691648859755; forceNewPlan=false, retain=false 2023-08-10T06:30:35,374 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=125 updating hbase:meta row=25a1c5b32e608be67dca00df059e0053, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:30:35,376 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE; OpenRegionProcedure 25a1c5b32e608be67dca00df059e0053, server=jenkins-hbase3.apache.org,45101,1691648859755}] 2023-08-10T06:30:35,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=124 2023-08-10T06:30:35,530 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(130): Open ns2:test-16916488687341,,1691649034386.25a1c5b32e608be67dca00df059e0053. 2023-08-10T06:30:35,530 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7254): Opening region: {ENCODED => 25a1c5b32e608be67dca00df059e0053, NAME => 'ns2:test-16916488687341,,1691649034386.25a1c5b32e608be67dca00df059e0053.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:30:35,531 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:30:35,531 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16916488687341 25a1c5b32e608be67dca00df059e0053 2023-08-10T06:30:35,531 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(888): Instantiated ns2:test-16916488687341,,1691649034386.25a1c5b32e608be67dca00df059e0053.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:30:35,531 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7296): checking encryption for 25a1c5b32e608be67dca00df059e0053 2023-08-10T06:30:35,531 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7299): checking classloading for 25a1c5b32e608be67dca00df059e0053 2023-08-10T06:30:35,532 INFO [StoreOpener-25a1c5b32e608be67dca00df059e0053-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 25a1c5b32e608be67dca00df059e0053 2023-08-10T06:30:35,534 INFO [StoreOpener-25a1c5b32e608be67dca00df059e0053-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 25a1c5b32e608be67dca00df059e0053 columnFamilyName f 2023-08-10T06:30:35,534 DEBUG [StoreOpener-25a1c5b32e608be67dca00df059e0053-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:30:35,534 INFO [StoreOpener-25a1c5b32e608be67dca00df059e0053-1 {}] regionserver.HStore(324): Store=25a1c5b32e608be67dca00df059e0053/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:30:35,535 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/25a1c5b32e608be67dca00df059e0053 2023-08-10T06:30:35,535 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/25a1c5b32e608be67dca00df059e0053 2023-08-10T06:30:35,537 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1076): writing seq id for 25a1c5b32e608be67dca00df059e0053 2023-08-10T06:30:35,538 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/25a1c5b32e608be67dca00df059e0053/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:30:35,539 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1093): Opened 25a1c5b32e608be67dca00df059e0053; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11269427360, jitterRate=0.049547210335731506}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-08-10T06:30:35,540 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(992): Region open journal for 25a1c5b32e608be67dca00df059e0053: 2023-08-10T06:30:35,540 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2552): Post open deploy tasks for ns2:test-16916488687341,,1691649034386.25a1c5b32e608be67dca00df059e0053., pid=126, masterSystemTime=1691649035528 2023-08-10T06:30:35,542 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2579): Finished post open deploy task for ns2:test-16916488687341,,1691649034386.25a1c5b32e608be67dca00df059e0053. 2023-08-10T06:30:35,542 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(158): Opened ns2:test-16916488687341,,1691649034386.25a1c5b32e608be67dca00df059e0053. 2023-08-10T06:30:35,542 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=125 updating hbase:meta row=25a1c5b32e608be67dca00df059e0053, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:30:35,545 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=126, resume processing ppid=125 2023-08-10T06:30:35,545 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=126, ppid=125, state=SUCCESS; OpenRegionProcedure 25a1c5b32e608be67dca00df059e0053, server=jenkins-hbase3.apache.org,45101,1691648859755 in 167 msec 2023-08-10T06:30:35,547 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=125, resume processing ppid=124 2023-08-10T06:30:35,547 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=125, ppid=124, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-16916488687341, region=25a1c5b32e608be67dca00df059e0053, ASSIGN in 325 msec 2023-08-10T06:30:35,547 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16916488687341","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649035547"}]},"ts":"1691649035547"} 2023-08-10T06:30:35,548 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16916488687341, state=ENABLED in hbase:meta 2023-08-10T06:30:35,551 DEBUG [PEWorker-4 {}] procedure.TruncateTableProcedure(145): truncate 'ns2:test-16916488687341' completed 2023-08-10T06:30:35,552 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=124, state=SUCCESS; TruncateTableProcedure (table=ns2:test-16916488687341 preserveSplits=true) in 1.1820 sec 2023-08-10T06:30:36,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=124 2023-08-10T06:30:36,483 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:45379/backupUT/backup_1691649017271/ns2/test-16916488687341/archive/data/ns2/test-16916488687341/4897eee4660ec19a1cc212f00aac000c from hbase tables ns2:test-16916488687341 to tables ns2:test-16916488687341 2023-08-10T06:30:36,483 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceRestoreJob(72): Restore ns2:test-16916488687341 into ns2:test-16916488687341 2023-08-10T06:30:36,485 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-ns2-test-16916488687341-1691649036483 from hdfs://localhost:45379/backupUT/backup_1691649017271/ns2/test-16916488687341/archive/data/ns2/test-16916488687341/4897eee4660ec19a1cc212f00aac000c 2023-08-10T06:30:36,489 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@684e4ebc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:30:36,490 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:30:36,491 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:49280, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:30:36,492 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-08-10T06:30:36,492 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:test-16916488687341 2023-08-10T06:30:36,495 DEBUG [hconnection-0x4d4449ac-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:30:36,496 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:55200, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:30:36,497 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:test-16916488687341: 2023-08-10T06:30:36,497 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-08-10T06:30:36,497 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/partitions_f679d74d-0117-4288-b6fb-fae1ca2a8c8a 2023-08-10T06:30:36,903 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:36,904 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:36,905 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:36,905 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:37,873 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/hadoop-6593025319153291710.jar 2023-08-10T06:30:37,873 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:37,873 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:37,874 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:37,874 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:37,875 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:37,875 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:37,875 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:37,875 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:37,876 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:37,876 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:30:37,877 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-08-10T06:30:37,877 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-08-10T06:30:37,878 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-08-10T06:30:37,878 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-08-10T06:30:37,879 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-08-10T06:30:37,879 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-08-10T06:30:37,879 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-08-10T06:30:37,880 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-08-10T06:30:37,880 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-08-10T06:30:37,880 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-08-10T06:30:37,881 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-08-10T06:30:37,881 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:37,881 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:37,882 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:37,882 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:37,882 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:37,883 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:37,883 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:30:37,883 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: ns2:test-16916488687341 2023-08-10T06:30:37,883 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:30:37,884 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-08-10T06:30:37,884 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-08-10T06:30:37,884 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:30:37,895 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-08-10T06:30:37,971 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691649037936/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-08-10T06:30:37,971 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691649037936/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-08-10T06:30:38,019 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:45379/backupUT/backup_1691649017271/ns2/test-16916488687341/archive/data/ns2/test-16916488687341/4897eee4660ec19a1cc212f00aac000c/f/a272e7ab9ac64bbb812d9b65df841988_SeqId_4_ 2023-08-10T06:30:38,022 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-08-10T06:30:38,025 DEBUG [LocalJobRunner Map Task Executor #0 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a272e7ab9ac64bbb812d9b65df841988_SeqId_4_ 2023-08-10T06:30:38,047 DEBUG [pool-438-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@743f0c38, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:30:38,055 DEBUG [hconnection-0x237d67a2-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:30:38,057 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:55216, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:30:38,060 DEBUG [pool-438-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:30:38,061 DEBUG [pool-438-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-08-10T06:30:38,061 DEBUG [pool-438-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-08-10T06:30:38,067 DEBUG [pool-438-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-ns2-test-16916488687341-1691649036483/_temporary/0/_temporary/attempt_local1424267052_0019_r_000000_0/f/85ddd8dc3e7d4845a2bcfe3b052901f1 is 37, key is row10/f:q1/1691648873613/Put/seqid=0 2023-08-10T06:30:38,068 WARN [Thread-2096 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:45101] Specified favored nodes: [jenkins-hbase3.apache.org:45101] 2023-08-10T06:30:39,019 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-ns2-test-16916488687341-1691649036483 2023-08-10T06:30:39,023 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e66bccf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:30:39,025 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:30:39,026 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:55218, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:30:39,031 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-ns2-test-16916488687341-1691649036483/_SUCCESS 2023-08-10T06:30:39,035 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:30:39,036 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:49290, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:30:39,046 DEBUG [LoadIncrementalHFiles-0 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 85ddd8dc3e7d4845a2bcfe3b052901f1 2023-08-10T06:30:39,046 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-ns2-test-16916488687341-1691649036483/f/85ddd8dc3e7d4845a2bcfe3b052901f1 first=Optional[row0] last=Optional[row98] 2023-08-10T06:30:39,047 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=ns2:test-16916488687341,,1691649034386.25a1c5b32e608be67dca00df059e0053., hostname=jenkins-hbase3.apache.org,45101,1691648859755, seqNum=2 for row with hfile group [{f,hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-ns2-test-16916488687341-1691649036483/f/85ddd8dc3e7d4845a2bcfe3b052901f1}] 2023-08-10T06:30:39,052 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-ns2-test-16916488687341-1691649036483/f/85ddd8dc3e7d4845a2bcfe3b052901f1 for inclusion in 25a1c5b32e608be67dca00df059e0053/f 2023-08-10T06:30:39,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-08-10T06:30:39,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HStore(625): Region bounds: first= last= 2023-08-10T06:30:39,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 85ddd8dc3e7d4845a2bcfe3b052901f1 2023-08-10T06:30:39,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HRegion(2520): Flush status journal for 25a1c5b32e608be67dca00df059e0053: 2023-08-10T06:30:39,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-ns2-test-16916488687341-1691649036483/f/85ddd8dc3e7d4845a2bcfe3b052901f1 to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/staging/jenkins__ns2_test-16916488687341__lm314smtg5c0igcr2lbo53p8h3pv331knkkteo8fkt0tgnpkpvmv67pu0f445u3a/f/85ddd8dc3e7d4845a2bcfe3b052901f1 2023-08-10T06:30:39,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/staging/jenkins__ns2_test-16916488687341__lm314smtg5c0igcr2lbo53p8h3pv331knkkteo8fkt0tgnpkpvmv67pu0f445u3a/f/85ddd8dc3e7d4845a2bcfe3b052901f1 as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/25a1c5b32e608be67dca00df059e0053/f/06cc12c6a1d04d99aa771c07eb41201c_SeqId_4_ 2023-08-10T06:30:39,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@306d9f02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:30:39,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:30:39,065 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:49298, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:30:39,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:30:39,069 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:55230, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:30:39,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] impl.BackupSystemTable(408): write bulk load descriptor to backup ns2:test-16916488687341 with 1 entries 2023-08-10T06:30:39,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] impl.BackupSystemTable(1582): writing raw bulk path hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/25a1c5b32e608be67dca00df059e0053/f/06cc12c6a1d04d99aa771c07eb41201c_SeqId_4_ for ns2:test-16916488687341 25a1c5b32e608be67dca00df059e0053 2023-08-10T06:30:39,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] impl.BackupSystemTable(415): written 1 rows for bulk load of ns2:test-16916488687341 2023-08-10T06:30:39,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:30:39,084 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/staging/jenkins__ns2_test-16916488687341__lm314smtg5c0igcr2lbo53p8h3pv331knkkteo8fkt0tgnpkpvmv67pu0f445u3a/f/85ddd8dc3e7d4845a2bcfe3b052901f1 into 25a1c5b32e608be67dca00df059e0053/f as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/25a1c5b32e608be67dca00df059e0053/f/06cc12c6a1d04d99aa771c07eb41201c_SeqId_4_ - updating store file list. 2023-08-10T06:30:39,088 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/25a1c5b32e608be67dca00df059e0053/f/06cc12c6a1d04d99aa771c07eb41201c_SeqId_4_ into 25a1c5b32e608be67dca00df059e0053/f 2023-08-10T06:30:39,088 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/staging/jenkins__ns2_test-16916488687341__lm314smtg5c0igcr2lbo53p8h3pv331knkkteo8fkt0tgnpkpvmv67pu0f445u3a/f/85ddd8dc3e7d4845a2bcfe3b052901f1 into 25a1c5b32e608be67dca00df059e0053/f (new location: hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/25a1c5b32e608be67dca00df059e0053/f/06cc12c6a1d04d99aa771c07eb41201c_SeqId_4_) 2023-08-10T06:30:39,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/staging/jenkins__ns2_test-16916488687341__lm314smtg5c0igcr2lbo53p8h3pv331knkkteo8fkt0tgnpkpvmv67pu0f445u3a/f/85ddd8dc3e7d4845a2bcfe3b052901f1 2023-08-10T06:30:39,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dcf8ab1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:30:39,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:30:39,097 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:49300, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-08-10T06:30:39,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:30:39,100 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:55242, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-08-10T06:30:39,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] impl.BackupSystemTable(388): write bulk load descriptor to backup ns2:test-16916488687341 with 1 entries 2023-08-10T06:30:39,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] impl.BackupSystemTable(1503): writing done bulk path hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/25a1c5b32e608be67dca00df059e0053/f/06cc12c6a1d04d99aa771c07eb41201c_SeqId_4_ for ns2:test-16916488687341 25a1c5b32e608be67dca00df059e0053 2023-08-10T06:30:39,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] impl.BackupSystemTable(394): written 1 rows for bulk load of ns2:test-16916488687341 2023-08-10T06:30:39,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:30:39,117 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:30:39,117 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-08-10T06:30:39,117 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-08-10T06:30:39,118 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(233): Backup: backup_1691649017271 hdfs://localhost:45379/backupUT/backup_1691649017271/ns2/test-16916488687341/ 2023-08-10T06:30:39,118 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:45379/backupUT/backup_1691649017271/.backup.manifest 2023-08-10T06:30:39,120 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649017271/.backup.manifest 2023-08-10T06:30:39,120 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(148): Restoring 'ns3:test-16916488687342' to 'ns3:test-16916488687342' from full backup image hdfs://localhost:45379/backupUT/backup_1691649017271/ns3/test-16916488687342 2023-08-10T06:30:39,125 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] util.RestoreTool(101): Folder tableArchivePath: hdfs://localhost:45379/backupUT/backup_1691649017271/ns3/test-16916488687342/archive/data/ns3/test-16916488687342 does not exists 2023-08-10T06:30:39,125 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] util.RestoreTool(326): find table descriptor but no archive dir for table ns3:test-16916488687342, will only create table 2023-08-10T06:30:39,126 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] util.RestoreTool(476): Truncating exising target table 'ns3:test-16916488687342', preserving region splits 2023-08-10T06:30:39,126 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster$12(2663): Client=jenkins//172.31.12.81 disable ns3:test-16916488687342 2023-08-10T06:30:39,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=127, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=ns3:test-16916488687342 2023-08-10T06:30:39,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=127 2023-08-10T06:30:39,129 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16916488687342","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649039129"}]},"ts":"1691649039129"} 2023-08-10T06:30:39,130 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16916488687342, state=DISABLING in hbase:meta 2023-08-10T06:30:39,132 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(293): Set ns3:test-16916488687342 to state=DISABLING 2023-08-10T06:30:39,132 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns3:test-16916488687342, region=77e10d7389ad2c7a8dfcbed4a1e8d4af, UNASSIGN}] 2023-08-10T06:30:39,133 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=128, ppid=127, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns3:test-16916488687342, region=77e10d7389ad2c7a8dfcbed4a1e8d4af, UNASSIGN 2023-08-10T06:30:39,134 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=128 updating hbase:meta row=77e10d7389ad2c7a8dfcbed4a1e8d4af, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:30:39,135 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=129, ppid=128, state=RUNNABLE; CloseRegionProcedure 77e10d7389ad2c7a8dfcbed4a1e8d4af, server=jenkins-hbase3.apache.org,45101,1691648859755}] 2023-08-10T06:30:39,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=127 2023-08-10T06:30:39,286 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] handler.UnassignRegionHandler(119): Close 77e10d7389ad2c7a8dfcbed4a1e8d4af 2023-08-10T06:30:39,286 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(1668): Closing 77e10d7389ad2c7a8dfcbed4a1e8d4af, disabling compactions & flushes 2023-08-10T06:30:39,287 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(1690): Closing region ns3:test-16916488687342,,1691648905436.77e10d7389ad2c7a8dfcbed4a1e8d4af. 2023-08-10T06:30:39,287 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16916488687342,,1691648905436.77e10d7389ad2c7a8dfcbed4a1e8d4af. 2023-08-10T06:30:39,287 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16916488687342,,1691648905436.77e10d7389ad2c7a8dfcbed4a1e8d4af. after waiting 0 ms 2023-08-10T06:30:39,287 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16916488687342,,1691648905436.77e10d7389ad2c7a8dfcbed4a1e8d4af. 2023-08-10T06:30:39,290 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns3/test-16916488687342/77e10d7389ad2c7a8dfcbed4a1e8d4af/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=1 2023-08-10T06:30:39,290 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:30:39,291 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(1909): Closed ns3:test-16916488687342,,1691648905436.77e10d7389ad2c7a8dfcbed4a1e8d4af. 2023-08-10T06:30:39,291 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(1622): Region close journal for 77e10d7389ad2c7a8dfcbed4a1e8d4af: 2023-08-10T06:30:39,293 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] handler.UnassignRegionHandler(163): Closed 77e10d7389ad2c7a8dfcbed4a1e8d4af 2023-08-10T06:30:39,293 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=128 updating hbase:meta row=77e10d7389ad2c7a8dfcbed4a1e8d4af, regionState=CLOSED 2023-08-10T06:30:39,296 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=129, resume processing ppid=128 2023-08-10T06:30:39,296 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=129, ppid=128, state=SUCCESS; CloseRegionProcedure 77e10d7389ad2c7a8dfcbed4a1e8d4af, server=jenkins-hbase3.apache.org,45101,1691648859755 in 159 msec 2023-08-10T06:30:39,297 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=128, resume processing ppid=127 2023-08-10T06:30:39,297 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=128, ppid=127, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-16916488687342, region=77e10d7389ad2c7a8dfcbed4a1e8d4af, UNASSIGN in 164 msec 2023-08-10T06:30:39,298 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16916488687342","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649039298"}]},"ts":"1691649039298"} 2023-08-10T06:30:39,299 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16916488687342, state=DISABLED in hbase:meta 2023-08-10T06:30:39,302 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(305): Set ns3:test-16916488687342 to state=DISABLED 2023-08-10T06:30:39,303 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=127, state=SUCCESS; DisableTableProcedure table=ns3:test-16916488687342 in 176 msec 2023-08-10T06:30:39,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=127 2023-08-10T06:30:39,431 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster$6(2479): Client=jenkins//172.31.12.81 truncate ns3:test-16916488687342 2023-08-10T06:30:39,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=130, state=RUNNABLE:TRUNCATE_TABLE_PRE_OPERATION; TruncateTableProcedure (table=ns3:test-16916488687342 preserveSplits=true) 2023-08-10T06:30:39,434 DEBUG [PEWorker-5 {}] procedure.TruncateTableProcedure(87): waiting for 'ns3:test-16916488687342' regions in transition 2023-08-10T06:30:39,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=130 2023-08-10T06:30:39,436 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns3/test-16916488687342/77e10d7389ad2c7a8dfcbed4a1e8d4af 2023-08-10T06:30:39,438 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns3/test-16916488687342/77e10d7389ad2c7a8dfcbed4a1e8d4af/f, FileablePath, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns3/test-16916488687342/77e10d7389ad2c7a8dfcbed4a1e8d4af/recovered.edits] 2023-08-10T06:30:39,442 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns3/test-16916488687342/77e10d7389ad2c7a8dfcbed4a1e8d4af/recovered.edits/7.seqid to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/archive/data/ns3/test-16916488687342/77e10d7389ad2c7a8dfcbed4a1e8d4af/recovered.edits/7.seqid 2023-08-10T06:30:39,443 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns3/test-16916488687342/77e10d7389ad2c7a8dfcbed4a1e8d4af 2023-08-10T06:30:39,443 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(293): Archived ns3:test-16916488687342 regions 2023-08-10T06:30:39,446 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of ns3:test-16916488687342 from hbase:meta 2023-08-10T06:30:39,448 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(387): Removing 'ns3:test-16916488687342' descriptor. 2023-08-10T06:30:39,448 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(377): Removing 'ns3:test-16916488687342' from region states. 2023-08-10T06:30:39,448 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns3:test-16916488687342,,1691648905436.77e10d7389ad2c7a8dfcbed4a1e8d4af.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1691649039448"}]},"ts":"9223372036854775807"} 2023-08-10T06:30:39,450 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-08-10T06:30:39,450 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 77e10d7389ad2c7a8dfcbed4a1e8d4af, NAME => 'ns3:test-16916488687342,,1691648905436.77e10d7389ad2c7a8dfcbed4a1e8d4af.', STARTKEY => '', ENDKEY => ''}] 2023-08-10T06:30:39,450 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(381): Marking 'ns3:test-16916488687342' as deleted. 2023-08-10T06:30:39,450 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns3:test-16916488687342","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1691649039450"}]},"ts":"9223372036854775807"} 2023-08-10T06:30:39,451 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1678): Deleted table ns3:test-16916488687342 state from META 2023-08-10T06:30:39,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=130 2023-08-10T06:30:39,733 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-08-10T06:30:39,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=130 2023-08-10T06:30:39,859 INFO [RegionOpenAndInit-ns3:test-16916488687342-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 547cc5444af6a8eb48a29d4d03cf49dd, NAME => 'ns3:test-16916488687342,,1691649039444.547cc5444af6a8eb48a29d4d03cf49dd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-16916488687342', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e 2023-08-10T06:30:40,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=130 2023-08-10T06:30:40,265 DEBUG [RegionOpenAndInit-ns3:test-16916488687342-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:test-16916488687342,,1691649039444.547cc5444af6a8eb48a29d4d03cf49dd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:30:40,265 DEBUG [RegionOpenAndInit-ns3:test-16916488687342-pool-0 {}] regionserver.HRegion(1668): Closing 547cc5444af6a8eb48a29d4d03cf49dd, disabling compactions & flushes 2023-08-10T06:30:40,265 INFO [RegionOpenAndInit-ns3:test-16916488687342-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:test-16916488687342,,1691649039444.547cc5444af6a8eb48a29d4d03cf49dd. 2023-08-10T06:30:40,265 DEBUG [RegionOpenAndInit-ns3:test-16916488687342-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16916488687342,,1691649039444.547cc5444af6a8eb48a29d4d03cf49dd. 2023-08-10T06:30:40,265 DEBUG [RegionOpenAndInit-ns3:test-16916488687342-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16916488687342,,1691649039444.547cc5444af6a8eb48a29d4d03cf49dd. after waiting 0 ms 2023-08-10T06:30:40,265 DEBUG [RegionOpenAndInit-ns3:test-16916488687342-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16916488687342,,1691649039444.547cc5444af6a8eb48a29d4d03cf49dd. 2023-08-10T06:30:40,265 INFO [RegionOpenAndInit-ns3:test-16916488687342-pool-0 {}] regionserver.HRegion(1909): Closed ns3:test-16916488687342,,1691649039444.547cc5444af6a8eb48a29d4d03cf49dd. 2023-08-10T06:30:40,265 DEBUG [RegionOpenAndInit-ns3:test-16916488687342-pool-0 {}] regionserver.HRegion(1622): Region close journal for 547cc5444af6a8eb48a29d4d03cf49dd: 2023-08-10T06:30:40,267 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:test-16916488687342,,1691649039444.547cc5444af6a8eb48a29d4d03cf49dd.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1691649040266"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691649040266"}]},"ts":"1691649040266"} 2023-08-10T06:30:40,268 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:30:40,269 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16916488687342","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649040269"}]},"ts":"1691649040269"} 2023-08-10T06:30:40,270 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16916488687342, state=ENABLING in hbase:meta 2023-08-10T06:30:40,278 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16916488687342, region=547cc5444af6a8eb48a29d4d03cf49dd, ASSIGN}] 2023-08-10T06:30:40,279 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16916488687342, region=547cc5444af6a8eb48a29d4d03cf49dd, ASSIGN 2023-08-10T06:30:40,280 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:test-16916488687342, region=547cc5444af6a8eb48a29d4d03cf49dd, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45101,1691648859755; forceNewPlan=false, retain=false 2023-08-10T06:30:40,431 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=131 updating hbase:meta row=547cc5444af6a8eb48a29d4d03cf49dd, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:30:40,433 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE; OpenRegionProcedure 547cc5444af6a8eb48a29d4d03cf49dd, server=jenkins-hbase3.apache.org,45101,1691648859755}] 2023-08-10T06:30:40,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=130 2023-08-10T06:30:40,587 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] handler.AssignRegionHandler(130): Open ns3:test-16916488687342,,1691649039444.547cc5444af6a8eb48a29d4d03cf49dd. 2023-08-10T06:30:40,587 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(7254): Opening region: {ENCODED => 547cc5444af6a8eb48a29d4d03cf49dd, NAME => 'ns3:test-16916488687342,,1691649039444.547cc5444af6a8eb48a29d4d03cf49dd.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:30:40,588 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:30:40,588 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16916488687342 547cc5444af6a8eb48a29d4d03cf49dd 2023-08-10T06:30:40,588 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(888): Instantiated ns3:test-16916488687342,,1691649039444.547cc5444af6a8eb48a29d4d03cf49dd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:30:40,588 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(7296): checking encryption for 547cc5444af6a8eb48a29d4d03cf49dd 2023-08-10T06:30:40,588 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(7299): checking classloading for 547cc5444af6a8eb48a29d4d03cf49dd 2023-08-10T06:30:40,589 INFO [StoreOpener-547cc5444af6a8eb48a29d4d03cf49dd-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 547cc5444af6a8eb48a29d4d03cf49dd 2023-08-10T06:30:40,590 INFO [StoreOpener-547cc5444af6a8eb48a29d4d03cf49dd-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 547cc5444af6a8eb48a29d4d03cf49dd columnFamilyName f 2023-08-10T06:30:40,590 DEBUG [StoreOpener-547cc5444af6a8eb48a29d4d03cf49dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:30:40,591 INFO [StoreOpener-547cc5444af6a8eb48a29d4d03cf49dd-1 {}] regionserver.HStore(324): Store=547cc5444af6a8eb48a29d4d03cf49dd/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:30:40,591 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns3/test-16916488687342/547cc5444af6a8eb48a29d4d03cf49dd 2023-08-10T06:30:40,592 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns3/test-16916488687342/547cc5444af6a8eb48a29d4d03cf49dd 2023-08-10T06:30:40,593 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1076): writing seq id for 547cc5444af6a8eb48a29d4d03cf49dd 2023-08-10T06:30:40,595 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns3/test-16916488687342/547cc5444af6a8eb48a29d4d03cf49dd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:30:40,595 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1093): Opened 547cc5444af6a8eb48a29d4d03cf49dd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11051321120, jitterRate=0.02923448383808136}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-08-10T06:30:40,596 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(992): Region open journal for 547cc5444af6a8eb48a29d4d03cf49dd: 2023-08-10T06:30:40,596 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegionServer(2552): Post open deploy tasks for ns3:test-16916488687342,,1691649039444.547cc5444af6a8eb48a29d4d03cf49dd., pid=132, masterSystemTime=1691649040584 2023-08-10T06:30:40,598 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegionServer(2579): Finished post open deploy task for ns3:test-16916488687342,,1691649039444.547cc5444af6a8eb48a29d4d03cf49dd. 2023-08-10T06:30:40,598 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] handler.AssignRegionHandler(158): Opened ns3:test-16916488687342,,1691649039444.547cc5444af6a8eb48a29d4d03cf49dd. 2023-08-10T06:30:40,598 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=131 updating hbase:meta row=547cc5444af6a8eb48a29d4d03cf49dd, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:30:40,600 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=132, resume processing ppid=131 2023-08-10T06:30:40,600 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=132, ppid=131, state=SUCCESS; OpenRegionProcedure 547cc5444af6a8eb48a29d4d03cf49dd, server=jenkins-hbase3.apache.org,45101,1691648859755 in 166 msec 2023-08-10T06:30:40,601 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=131, resume processing ppid=130 2023-08-10T06:30:40,601 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=131, ppid=130, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-16916488687342, region=547cc5444af6a8eb48a29d4d03cf49dd, ASSIGN in 322 msec 2023-08-10T06:30:40,601 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16916488687342","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649040601"}]},"ts":"1691649040601"} 2023-08-10T06:30:40,602 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16916488687342, state=ENABLED in hbase:meta 2023-08-10T06:30:40,605 DEBUG [PEWorker-1 {}] procedure.TruncateTableProcedure(145): truncate 'ns3:test-16916488687342' completed 2023-08-10T06:30:40,606 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=130, state=SUCCESS; TruncateTableProcedure (table=ns3:test-16916488687342 preserveSplits=true) in 1.1740 sec 2023-08-10T06:30:41,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=130 2023-08-10T06:30:41,541 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-08-10T06:30:41,541 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(233): Backup: backup_1691649017271 hdfs://localhost:45379/backupUT/backup_1691649017271/ns3/test-16916488687342/ 2023-08-10T06:30:41,542 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-08-10T06:30:41,542 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:30:41,561 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreMultipleOverwriteCommand Thread=606 (was 601) - Thread LEAK? -, OpenFileDescriptor=680 (was 675) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=162 (was 133) - SystemLoadAverage LEAK? -, ProcessCount=171 (was 171), AvailableMemoryMB=1801 (was 1303) - AvailableMemoryMB LEAK? - 2023-08-10T06:30:41,561 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(130): Thread=606 is superior to 500 2023-08-10T06:30:41,579 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreMultipleCommand Thread=606, OpenFileDescriptor=680, MaxFileDescriptor=60000, SystemLoadAverage=162, ProcessCount=171, AvailableMemoryMB=1800 2023-08-10T06:30:41,579 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(130): Thread=606 is superior to 500 2023-08-10T06:30:41,579 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] backup.TestFullRestore(144): create full backup image on multiple tables: command-line 2023-08-10T06:30:41,585 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f1c787, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:30:41,588 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:30:41,590 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:43112, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:30:41,592 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:30:41,593 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:57776, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:30:41,604 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-08-10T06:30:41,609 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:30:41,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691649041609 (current time:1691649041609). 2023-08-10T06:30:41,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:30:41,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-08-10T06:30:41,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44a1ae8e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:30:41,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:30:41,616 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:43122, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:30:41,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:30:41,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:30:41,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=133, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:30:41,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 133 2023-08-10T06:30:41,619 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:30:41,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=133 2023-08-10T06:30:41,620 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:30:41,622 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:30:41,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=133 2023-08-10T06:30:41,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=133 2023-08-10T06:30:42,007 DEBUG [regionserver/jenkins-hbase3:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.46153846153846156, tune throughput to 73.08 MB/second 2023-08-10T06:30:42,030 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:30:42,030 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72}] 2023-08-10T06:30:42,032 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=134, ppid=133, state=RUNNABLE; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:30:42,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45101 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=134 2023-08-10T06:30:42,183 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. 2023-08-10T06:30:42,184 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.HRegion(2819): Flushing b9c493f2c9fbef45de19e8268a534e72 2/2 column families, dataSize=1.50 KB heapSize=2.73 KB 2023-08-10T06:30:42,187 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/meta/8f5b1bc0de87458e96bd38212599dcbd is 171, key is trslm:hdfs://localhost:45379/backupUT\x00ns2:test-16916488687341/meta:log-roll-map/1691649033174/Put/seqid=0 2023-08-10T06:30:42,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=133 2023-08-10T06:30:42,592 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=717 B at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/meta/8f5b1bc0de87458e96bd38212599dcbd 2023-08-10T06:30:42,597 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 8f5b1bc0de87458e96bd38212599dcbd 2023-08-10T06:30:42,600 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/session/3702fe0322c94bcc85372b5eb9c99029 is 524, key is session:backup_1691649017271/session:context/1691649034011/Put/seqid=0 2023-08-10T06:30:42,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=133 2023-08-10T06:30:43,008 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=820 B at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/session/3702fe0322c94bcc85372b5eb9c99029 2023-08-10T06:30:43,013 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 3702fe0322c94bcc85372b5eb9c99029 2023-08-10T06:30:43,014 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/meta/8f5b1bc0de87458e96bd38212599dcbd as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/8f5b1bc0de87458e96bd38212599dcbd 2023-08-10T06:30:43,019 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/8f5b1bc0de87458e96bd38212599dcbd, entries=6, sequenceid=105, filesize=5.8 K 2023-08-10T06:30:43,019 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/session/3702fe0322c94bcc85372b5eb9c99029 as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/3702fe0322c94bcc85372b5eb9c99029 2023-08-10T06:30:43,025 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/3702fe0322c94bcc85372b5eb9c99029, entries=2, sequenceid=105, filesize=5.5 K 2023-08-10T06:30:43,026 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.HRegion(3022): Finished flush of dataSize ~1.50 KB/1537, heapSize ~2.70 KB/2760, currentSize=0 B/0 for b9c493f2c9fbef45de19e8268a534e72 in 843ms, sequenceid=105, compaction requested=true 2023-08-10T06:30:43,026 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.HRegion(2520): Flush status journal for b9c493f2c9fbef45de19e8268a534e72: 2023-08-10T06:30:43,026 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. for snapshot_backup_system completed. 2023-08-10T06:30:43,026 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72.' region-info for snapshot=snapshot_backup_system 2023-08-10T06:30:43,026 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:30:43,027 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/bc10682e7f36491d941e2fe62e2e4100, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/9d9a4feb1c4e44d99cc63e3f68878731, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/f04a99990b2c429eaf1096ef413be3b8, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/9e97dc38b66b476f952990b23fa0a3ca, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/930c578fe3f543e1b6308abfb31c7246, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/24713cefebe0499c812c6d209dea0fd6, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/0ae449898702485faa63081b0f3f028a, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/cec52c43e8824ebfbbc2cf17b7a9ec5f, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/8f5b1bc0de87458e96bd38212599dcbd] hfiles 2023-08-10T06:30:43,027 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (1/9): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/bc10682e7f36491d941e2fe62e2e4100 for snapshot=snapshot_backup_system 2023-08-10T06:30:43,027 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (2/9): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/9d9a4feb1c4e44d99cc63e3f68878731 for snapshot=snapshot_backup_system 2023-08-10T06:30:43,027 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (3/9): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/f04a99990b2c429eaf1096ef413be3b8 for snapshot=snapshot_backup_system 2023-08-10T06:30:43,027 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (4/9): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/9e97dc38b66b476f952990b23fa0a3ca for snapshot=snapshot_backup_system 2023-08-10T06:30:43,027 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (5/9): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/930c578fe3f543e1b6308abfb31c7246 for snapshot=snapshot_backup_system 2023-08-10T06:30:43,027 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (6/9): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/24713cefebe0499c812c6d209dea0fd6 for snapshot=snapshot_backup_system 2023-08-10T06:30:43,027 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (7/9): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/0ae449898702485faa63081b0f3f028a for snapshot=snapshot_backup_system 2023-08-10T06:30:43,027 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (8/9): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/cec52c43e8824ebfbbc2cf17b7a9ec5f for snapshot=snapshot_backup_system 2023-08-10T06:30:43,027 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (9/9): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/8f5b1bc0de87458e96bd38212599dcbd for snapshot=snapshot_backup_system 2023-08-10T06:30:43,027 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/e88678d5b4b14eecabac817daf12539e, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/37561b9a8ae2406bb6d053c33aeff31c, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/883911c19cab473f8bf801f934c90bc1, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/0c7705a45c554d6a97fa37fac319ceff, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/398a15bfffb84acfadb1be252c2e133a, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/adceb6320b02451abd02beed6c69c000, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/68fd0cf6c3094cc7af1a5176829fd9a4, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/40f9fbdeac5b49dbb654e429c177708d, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/75226fb0202e42819224b5e5920a02f2, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/3702fe0322c94bcc85372b5eb9c99029] hfiles 2023-08-10T06:30:43,027 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (1/10): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/e88678d5b4b14eecabac817daf12539e for snapshot=snapshot_backup_system 2023-08-10T06:30:43,027 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (2/10): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/37561b9a8ae2406bb6d053c33aeff31c for snapshot=snapshot_backup_system 2023-08-10T06:30:43,027 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (3/10): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/883911c19cab473f8bf801f934c90bc1 for snapshot=snapshot_backup_system 2023-08-10T06:30:43,027 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (4/10): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/0c7705a45c554d6a97fa37fac319ceff for snapshot=snapshot_backup_system 2023-08-10T06:30:43,027 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (5/10): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/398a15bfffb84acfadb1be252c2e133a for snapshot=snapshot_backup_system 2023-08-10T06:30:43,027 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (6/10): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/adceb6320b02451abd02beed6c69c000 for snapshot=snapshot_backup_system 2023-08-10T06:30:43,027 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (7/10): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/68fd0cf6c3094cc7af1a5176829fd9a4 for snapshot=snapshot_backup_system 2023-08-10T06:30:43,027 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (8/10): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/40f9fbdeac5b49dbb654e429c177708d for snapshot=snapshot_backup_system 2023-08-10T06:30:43,027 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (9/10): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/75226fb0202e42819224b5e5920a02f2 for snapshot=snapshot_backup_system 2023-08-10T06:30:43,027 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (10/10): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/3702fe0322c94bcc85372b5eb9c99029 for snapshot=snapshot_backup_system 2023-08-10T06:30:43,442 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. 2023-08-10T06:30:43,442 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=134 2023-08-10T06:30:43,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster(3994): Remote procedure done, pid=134 2023-08-10T06:30:43,443 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:30:43,444 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=134, ppid=133, state=RUNNABLE; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:30:43,446 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=134, resume processing ppid=133 2023-08-10T06:30:43,446 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=134, ppid=133, state=SUCCESS; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72 in 1.4150 sec 2023-08-10T06:30:43,446 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:30:43,446 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:30:43,447 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:30:43,447 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-08-10T06:30:43,447 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_backup_system 2023-08-10T06:30:43,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=133 2023-08-10T06:30:43,859 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:30:43,870 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:30:43,870 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_backup_system 2023-08-10T06:30:44,686 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:30:44,686 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 133 2023-08-10T06:30:44,687 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=133, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 3.0690 sec 2023-08-10T06:30:44,792 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-08-10T06:30:45,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=133 2023-08-10T06:30:45,725 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(115): Backup backup_1691649041585 started at 1691649045725. 2023-08-10T06:30:45,727 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(119): Backup session backup_1691649041585 has been started. 2023-08-10T06:30:45,732 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-08-10T06:30:45,733 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-08-10T06:30:45,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-08-10T06:30:45,733 INFO [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-08-10T06:30:45,733 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-08-10T06:30:45,734 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-08-10T06:30:45,734 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-08-10T06:30:45,734 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:30:45,737 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:30:45,737 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:30:45,737 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:30:45,737 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-08-10T06:30:45,737 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-08-10T06:30:45,738 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-08-10T06:30:45,738 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:30:45,738 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-08-10T06:30:45,738 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-08-10T06:30:45,738 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-08-10T06:30:45,738 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-08-10T06:30:45,738 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-08-10T06:30:45,740 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-08-10T06:30:45,740 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-08-10T06:30:45,740 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,45101,1691648859755' joining acquired barrier for procedure (rolllog) in zk 2023-08-10T06:30:45,742 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-08-10T06:30:45,742 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:30:45,742 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:30:45,742 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:30:45,743 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-08-10T06:30:45,743 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:30:45,743 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:30:45,743 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:30:45,743 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:30:45,744 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:30:45,744 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,45101,1691648859755' joining acquired barrier for procedure 'rolllog' on coordinator 2023-08-10T06:30:45,744 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@38d3fdb7[Count = 0] remaining members to acquire global barrier 2023-08-10T06:30:45,744 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-08-10T06:30:45,744 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-08-10T06:30:45,745 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-08-10T06:30:45,745 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-08-10T06:30:45,745 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-08-10T06:30:45,745 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-08-10T06:30:45,745 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-08-10T06:30:45,745 DEBUG [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:30:45,745 INFO [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1691649021478 highest: 1691649021478 on jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:30:45,745 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta:.meta(num 1691649021448) roll requested 2023-08-10T06:30:45,757 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691649045746.meta, exclude list is [], retry=0 2023-08-10T06:30:45,760 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK] 2023-08-10T06:30:45,761 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691649021448.meta with entries=24, filesize=6.28 KB; new WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691649045746.meta 2023-08-10T06:30:45,761 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK]] 2023-08-10T06:30:45,761 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691649021448.meta is not closed yet, will try archiving it next time 2023-08-10T06:30:45,761 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C45101%2C1691648859755:(num 1691649021478) roll requested 2023-08-10T06:30:45,773 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691649045762, exclude list is [], retry=0 2023-08-10T06:30:45,777 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK] 2023-08-10T06:30:45,779 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691649021478 with entries=17, filesize=5.81 KB; new WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691649045762 2023-08-10T06:30:45,779 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK]] 2023-08-10T06:30:45,779 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691649021478 is not closed yet, will try archiving it next time 2023-08-10T06:30:45,779 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691648997529 to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/oldWALs/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691648997529 2023-08-10T06:30:45,786 DEBUG [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 41 2023-08-10T06:30:45,786 INFO [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1691649045762 on jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:30:45,793 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-08-10T06:30:45,793 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,45101,1691648859755' in zk 2023-08-10T06:30:45,796 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-08-10T06:30:45,796 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:30:45,796 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:30:45,796 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-08-10T06:30:45,796 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:30:45,797 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:30:45,797 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:30:45,797 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:30:45,798 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:30:45,798 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:30:45,798 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:30:45,799 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:30:45,799 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:30:45,799 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,45101,1691648859755': 2023-08-10T06:30:45,800 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,45101,1691648859755' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-08-10T06:30:45,800 INFO [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-08-10T06:30:45,800 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-08-10T06:30:45,800 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-08-10T06:30:45,800 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-08-10T06:30:45,800 INFO [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-08-10T06:30:45,803 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-08-10T06:30:45,803 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:30:45,803 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:30:45,803 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-08-10T06:30:45,803 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:30:45,803 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:30:45,803 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:30:45,803 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:30:45,803 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:30:45,803 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:30:45,804 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:30:45,804 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:30:45,805 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:30:45,805 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:30:45,805 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:30:45,806 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:30:45,810 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:30:45,810 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-08-10T06:30:45,810 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-08-10T06:30:45,811 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:30:45,811 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:30:45,811 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:30:45,811 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:30:45,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-08-10T06:30:45,914 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1691649045914_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } 2023-08-10T06:30:45,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691649045914 (current time:1691649045914). 2023-08-10T06:30:45,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:30:45,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1691649045914_ns2_test-16916488687341 VERSION not specified, setting to 2 2023-08-10T06:30:45,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@206d9936, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:30:45,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:30:45,924 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:43134, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:30:45,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:30:45,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:30:45,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=135, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1691649045914_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } 2023-08-10T06:30:45,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1691649045914_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 }, snapshot procedure id = 135 2023-08-10T06:30:45,927 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1691649045914_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:30:45,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=135 2023-08-10T06:30:45,927 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1691649045914_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:30:45,929 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1691649045914_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:30:46,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=135 2023-08-10T06:30:46,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=135 2023-08-10T06:30:46,334 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1691649045914_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:30:46,334 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE; SnapshotRegionProcedure 25a1c5b32e608be67dca00df059e0053}] 2023-08-10T06:30:46,336 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=136, ppid=135, state=RUNNABLE; SnapshotRegionProcedure 25a1c5b32e608be67dca00df059e0053 2023-08-10T06:30:46,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45101 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=136 2023-08-10T06:30:46,488 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-16916488687341,,1691649034386.25a1c5b32e608be67dca00df059e0053. 2023-08-10T06:30:46,488 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] regionserver.HRegion(2520): Flush status journal for 25a1c5b32e608be67dca00df059e0053: 2023-08-10T06:30:46,488 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-16916488687341,,1691649034386.25a1c5b32e608be67dca00df059e0053. for snapshot_1691649045914_ns2_test-16916488687341 completed. 2023-08-10T06:30:46,488 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] snapshot.SnapshotManifest(240): Storing 'ns2:test-16916488687341,,1691649034386.25a1c5b32e608be67dca00df059e0053.' region-info for snapshot=snapshot_1691649045914_ns2_test-16916488687341 2023-08-10T06:30:46,488 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:30:46,488 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/25a1c5b32e608be67dca00df059e0053/f/06cc12c6a1d04d99aa771c07eb41201c_SeqId_4_] hfiles 2023-08-10T06:30:46,488 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/25a1c5b32e608be67dca00df059e0053/f/06cc12c6a1d04d99aa771c07eb41201c_SeqId_4_ for snapshot=snapshot_1691649045914_ns2_test-16916488687341 2023-08-10T06:30:46,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=135 2023-08-10T06:30:46,894 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-16916488687341,,1691649034386.25a1c5b32e608be67dca00df059e0053. 2023-08-10T06:30:46,894 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=136 2023-08-10T06:30:46,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster(3994): Remote procedure done, pid=136 2023-08-10T06:30:46,894 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1691649045914_ns2_test-16916488687341 on region 25a1c5b32e608be67dca00df059e0053 2023-08-10T06:30:46,896 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=136, ppid=135, state=RUNNABLE; SnapshotRegionProcedure 25a1c5b32e608be67dca00df059e0053 2023-08-10T06:30:46,898 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=136, resume processing ppid=135 2023-08-10T06:30:46,898 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=136, ppid=135, state=SUCCESS; SnapshotRegionProcedure 25a1c5b32e608be67dca00df059e0053 in 562 msec 2023-08-10T06:30:46,898 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1691649045914_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:30:46,898 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1691649045914_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:30:46,899 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1691649045914_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:30:46,899 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1691649045914_ns2_test-16916488687341 2023-08-10T06:30:46,899 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_1691649045914_ns2_test-16916488687341 2023-08-10T06:30:46,907 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1691649045914_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:30:46,912 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1691649045914_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:30:46,912 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_1691649045914_ns2_test-16916488687341 to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691649045914_ns2_test-16916488687341 2023-08-10T06:30:47,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=135 2023-08-10T06:30:47,728 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1691649045914_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:30:47,728 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1691649045914_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 }, snapshot procedure id = 135 2023-08-10T06:30:47,729 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=135, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1691649045914_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } in 1.8030 sec 2023-08-10T06:30:48,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=135 2023-08-10T06:30:48,034 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1691649048033_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } 2023-08-10T06:30:48,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691649048034 (current time:1691649048034). 2023-08-10T06:30:48,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:30:48,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1691649048033_ns3_test-16916488687342 VERSION not specified, setting to 2 2023-08-10T06:30:48,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3448db3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:30:48,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:30:48,042 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:43142, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:30:48,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:30:48,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:30:48,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=137, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1691649048033_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } 2023-08-10T06:30:48,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1691649048033_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 }, snapshot procedure id = 137 2023-08-10T06:30:48,045 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1691649048033_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:30:48,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=137 2023-08-10T06:30:48,046 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1691649048033_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:30:48,047 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1691649048033_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:30:48,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=137 2023-08-10T06:30:48,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=137 2023-08-10T06:30:48,452 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1691649048033_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:30:48,453 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE; SnapshotRegionProcedure 547cc5444af6a8eb48a29d4d03cf49dd}] 2023-08-10T06:30:48,454 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=138, ppid=137, state=RUNNABLE; SnapshotRegionProcedure 547cc5444af6a8eb48a29d4d03cf49dd 2023-08-10T06:30:48,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45101 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=138 2023-08-10T06:30:48,606 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=138}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-16916488687342,,1691649039444.547cc5444af6a8eb48a29d4d03cf49dd. 2023-08-10T06:30:48,607 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=138}] regionserver.HRegion(2520): Flush status journal for 547cc5444af6a8eb48a29d4d03cf49dd: 2023-08-10T06:30:48,607 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=138}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-16916488687342,,1691649039444.547cc5444af6a8eb48a29d4d03cf49dd. for snapshot_1691649048033_ns3_test-16916488687342 completed. 2023-08-10T06:30:48,607 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=138}] snapshot.SnapshotManifest(240): Storing 'ns3:test-16916488687342,,1691649039444.547cc5444af6a8eb48a29d4d03cf49dd.' region-info for snapshot=snapshot_1691649048033_ns3_test-16916488687342 2023-08-10T06:30:48,607 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=138}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:30:48,607 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=138}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-08-10T06:30:48,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=137 2023-08-10T06:30:49,013 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=138}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-16916488687342,,1691649039444.547cc5444af6a8eb48a29d4d03cf49dd. 2023-08-10T06:30:49,013 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=138 2023-08-10T06:30:49,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster(3994): Remote procedure done, pid=138 2023-08-10T06:30:49,013 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1691649048033_ns3_test-16916488687342 on region 547cc5444af6a8eb48a29d4d03cf49dd 2023-08-10T06:30:49,015 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=138, ppid=137, state=RUNNABLE; SnapshotRegionProcedure 547cc5444af6a8eb48a29d4d03cf49dd 2023-08-10T06:30:49,016 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=138, resume processing ppid=137 2023-08-10T06:30:49,016 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=138, ppid=137, state=SUCCESS; SnapshotRegionProcedure 547cc5444af6a8eb48a29d4d03cf49dd in 562 msec 2023-08-10T06:30:49,016 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1691649048033_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:30:49,017 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1691649048033_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:30:49,017 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1691649048033_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:30:49,017 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1691649048033_ns3_test-16916488687342 2023-08-10T06:30:49,018 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_1691649048033_ns3_test-16916488687342 2023-08-10T06:30:49,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=137 2023-08-10T06:30:49,426 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1691649048033_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:30:49,430 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1691649048033_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:30:49,430 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_1691649048033_ns3_test-16916488687342 to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691649048033_ns3_test-16916488687342 2023-08-10T06:30:50,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=137 2023-08-10T06:30:50,245 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1691649048033_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:30:50,245 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1691649048033_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 }, snapshot procedure id = 137 2023-08-10T06:30:50,246 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=137, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1691649048033_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } in 2.2020 sec 2023-08-10T06:30:52,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=137 2023-08-10T06:30:52,151 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1691649041585 2023-08-10T06:30:52,151 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-08-10T06:30:52,151 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2023-08-10T06:30:52,151 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1691649041585_ns3:test-16916488687342 2023-08-10T06:30:52,151 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1691649048033_ns3_test-16916488687342 to hdfs://localhost:45379/backupUT/backup_1691649041585/ns3/test-16916488687342/ 2023-08-10T06:30:52,151 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-08-10T06:30:52,174 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:45379, inputRoot=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e 2023-08-10T06:30:52,174 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,33705,1691648856595_-1670772113_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:45379/backupUT/backup_1691649041585/ns3/test-16916488687342, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:45379/backupUT/backup_1691649041585/ns3/test-16916488687342/.hbase-snapshot/.tmp/snapshot_1691649048033_ns3_test-16916488687342 2023-08-10T06:30:52,174 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:45379, inputRoot=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e, snapshotDir=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691649048033_ns3_test-16916488687342. 2023-08-10T06:30:52,179 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691649048033_ns3_test-16916488687342 to hdfs://localhost:45379/backupUT/backup_1691649041585/ns3/test-16916488687342/.hbase-snapshot/.tmp/snapshot_1691649048033_ns3_test-16916488687342 2023-08-10T06:30:52,591 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:52,592 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:52,592 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:52,593 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:53,583 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/hadoop-5319890779153393131.jar 2023-08-10T06:30:53,584 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:53,584 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:53,584 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:53,585 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:53,585 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:53,585 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:53,586 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:53,586 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:53,587 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:53,587 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:30:53,588 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-08-10T06:30:53,588 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-08-10T06:30:53,589 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-08-10T06:30:53,589 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-08-10T06:30:53,590 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-08-10T06:30:53,590 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-08-10T06:30:53,590 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-08-10T06:30:53,591 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-08-10T06:30:53,591 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-08-10T06:30:53,591 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-08-10T06:30:53,592 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-08-10T06:30:53,592 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:30:53,593 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:30:53,593 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:30:53,593 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:30:53,594 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:30:53,594 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:30:53,595 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:30:53,639 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1691649048033_ns3_test-16916488687342' hfile list 2023-08-10T06:30:53,714 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691649053677/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-08-10T06:30:53,714 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691649053677/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-08-10T06:30:54,746 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-08-10T06:30:54,747 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-08-10T06:30:54,751 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1691649048033_ns3_test-16916488687342 2023-08-10T06:30:54,751 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1691649048033_ns3_test-16916488687342 finished. 2023-08-10T06:30:54,751 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1691649041585_ns2:test-16916488687341 2023-08-10T06:30:54,751 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1691649045914_ns2_test-16916488687341 to hdfs://localhost:45379/backupUT/backup_1691649041585/ns2/test-16916488687341/ 2023-08-10T06:30:54,751 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-08-10T06:30:54,775 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:45379, inputRoot=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e 2023-08-10T06:30:54,775 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,33705,1691648856595_1726376766_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:45379/backupUT/backup_1691649041585/ns2/test-16916488687341, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:45379/backupUT/backup_1691649041585/ns2/test-16916488687341/.hbase-snapshot/.tmp/snapshot_1691649045914_ns2_test-16916488687341 2023-08-10T06:30:54,776 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:45379, inputRoot=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e, snapshotDir=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691649045914_ns2_test-16916488687341. 2023-08-10T06:30:54,783 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691649045914_ns2_test-16916488687341 to hdfs://localhost:45379/backupUT/backup_1691649041585/ns2/test-16916488687341/.hbase-snapshot/.tmp/snapshot_1691649045914_ns2_test-16916488687341 2023-08-10T06:30:55,198 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:55,198 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:55,199 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:55,200 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:56,184 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/hadoop-52324316112104208.jar 2023-08-10T06:30:56,184 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:56,184 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:56,185 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:56,185 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:56,185 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:56,186 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:56,186 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:56,186 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:56,187 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-08-10T06:30:56,187 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:30:56,188 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-08-10T06:30:56,188 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-08-10T06:30:56,189 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-08-10T06:30:56,189 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-08-10T06:30:56,190 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-08-10T06:30:56,190 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-08-10T06:30:56,190 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-08-10T06:30:56,191 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-08-10T06:30:56,191 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-08-10T06:30:56,191 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-08-10T06:30:56,192 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-08-10T06:30:56,192 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:30:56,193 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:30:56,193 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:30:56,193 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:30:56,194 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:30:56,194 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:30:56,195 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:30:56,237 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1691649045914_ns2_test-16916488687341' hfile list 2023-08-10T06:30:56,239 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-08-10T06:30:56,305 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691649056272/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-08-10T06:30:56,305 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691649056272/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-08-10T06:30:56,356 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-08-10T06:30:56,379 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/25a1c5b32e608be67dca00df059e0053/f/06cc12c6a1d04d99aa771c07eb41201c_SeqId_4_ output=hdfs://localhost:45379/backupUT/backup_1691649041585/ns2/test-16916488687341/archive/data/ns2/test-16916488687341/25a1c5b32e608be67dca00df059e0053/f/06cc12c6a1d04d99aa771c07eb41201c_SeqId_4_ 2023-08-10T06:30:56,379 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8725 (8.5 K) time=0sec 8.321M/sec 2023-08-10T06:30:57,340 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-08-10T06:30:57,341 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-08-10T06:30:57,346 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1691649045914_ns2_test-16916488687341 2023-08-10T06:30:57,346 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1691649045914_ns2_test-16916488687341 finished. 2023-08-10T06:30:57,346 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(1014): ns3:test-16916488687342 2023-08-10T06:30:57,346 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(1014): ns2:test-16916488687341 2023-08-10T06:30:57,352 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691649041585 2023-08-10T06:30:57,352 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-08-10T06:30:57,757 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:45379/backupUT/backup_1691649041585/.backup.manifest 2023-08-10T06:30:57,757 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691649041585 2023-08-10T06:30:57,757 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-08-10T06:30:58,162 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:45379/backupUT/backup_1691649041585/.backup.manifest 2023-08-10T06:30:58,162 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(358): Backup backup_1691649041585 finished: type=FULL,tablelist=ns3:test-16916488687342;ns2:test-16916488687341,targetRootDir=hdfs://localhost:45379/backupUT,startts=1691649045725,completets=1691649057352,bytescopied=0 2023-08-10T06:30:58,162 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-08-10T06:30:58,163 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1691649048033_ns3_test-16916488687342 2023-08-10T06:30:58,163 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1691649048033_ns3_test-16916488687342" 2023-08-10T06:30:58,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1691649048033_ns3_test-16916488687342 2023-08-10T06:30:58,167 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1691649048033_ns3_test-16916488687342 for backup backup_1691649041585 succeeded. 2023-08-10T06:30:58,167 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1691649045914_ns2_test-16916488687341 2023-08-10T06:30:58,168 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1691649045914_ns2_test-16916488687341" 2023-08-10T06:30:58,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1691649045914_ns2_test-16916488687341 2023-08-10T06:30:58,170 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1691649045914_ns2_test-16916488687341 for backup backup_1691649041585 succeeded. 2023-08-10T06:30:58,171 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-08-10T06:30:58,175 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-08-10T06:30:58,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-08-10T06:30:58,177 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-08-10T06:30:58,178 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-08-10T06:30:58,184 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(376): Backup backup_1691649041585 completed. 2023-08-10T06:30:58,184 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:30:58,205 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a7faab7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:30:58,207 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:30:58,208 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:56438, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:30:58,209 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(101): HBase table ns2:table2 does not exist. It will be created during restore process 2023-08-10T06:30:58,210 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(101): HBase table ns3:table3_restore does not exist. It will be created during restore process 2023-08-10T06:30:58,210 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:45379/backupUT/backup_1691649041585/.backup.manifest 2023-08-10T06:30:58,212 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649041585/.backup.manifest 2023-08-10T06:30:58,212 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:45379/backupUT/backup_1691649041585/.backup.manifest 2023-08-10T06:30:58,214 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649041585/.backup.manifest 2023-08-10T06:30:58,214 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:45379/backupUT/backup_1691649041585/.backup.manifest 2023-08-10T06:30:58,216 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649041585/.backup.manifest 2023-08-10T06:30:58,216 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(148): Restoring 'ns2:test-16916488687341' to 'ns2:table2' from full backup image hdfs://localhost:45379/backupUT/backup_1691649041585/ns2/test-16916488687341 2023-08-10T06:30:58,222 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] util.RestoreTool(487): Creating target table 'ns2:table2' 2023-08-10T06:30:58,222 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] util.RestoreTool(391): Parsing region dir: hdfs://localhost:45379/backupUT/backup_1691649041585/ns2/test-16916488687341/archive/data/ns2/test-16916488687341/25a1c5b32e608be67dca00df059e0053 2023-08-10T06:30:58,223 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:45379/backupUT/backup_1691649041585/ns2/test-16916488687341/archive/data/ns2/test-16916488687341/25a1c5b32e608be67dca00df059e0053/f in region [hdfs://localhost:45379/backupUT/backup_1691649041585/ns2/test-16916488687341/archive/data/ns2/test-16916488687341/25a1c5b32e608be67dca00df059e0053] 2023-08-10T06:30:58,225 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] util.RestoreTool(442): Trying to figure out region boundaries hfile=hdfs://localhost:45379/backupUT/backup_1691649041585/ns2/test-16916488687341/archive/data/ns2/test-16916488687341/25a1c5b32e608be67dca00df059e0053/f/06cc12c6a1d04d99aa771c07eb41201c_SeqId_4_ first=row0 last=row98 2023-08-10T06:30:58,225 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 06cc12c6a1d04d99aa771c07eb41201c_SeqId_4_ 2023-08-10T06:30:58,226 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:30:58,228 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:51936, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:30:58,228 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'ns2:table2', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:30:58,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=139, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns2:table2 2023-08-10T06:30:58,231 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=139, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_PRE_OPERATION 2023-08-10T06:30:58,231 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns2" qualifier: "table2" procId is: 139 2023-08-10T06:30:58,231 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=139, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-08-10T06:30:58,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=139 2023-08-10T06:30:58,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=139 2023-08-10T06:30:58,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=139 2023-08-10T06:30:58,638 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 8adb09ea4b151f0bc51e6dbcbab134a4, NAME => 'ns2:table2,,1691649058228.8adb09ea4b151f0bc51e6dbcbab134a4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:table2', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e 2023-08-10T06:30:58,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=139 2023-08-10T06:30:59,043 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:table2,,1691649058228.8adb09ea4b151f0bc51e6dbcbab134a4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:30:59,044 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1668): Closing 8adb09ea4b151f0bc51e6dbcbab134a4, disabling compactions & flushes 2023-08-10T06:30:59,044 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:table2,,1691649058228.8adb09ea4b151f0bc51e6dbcbab134a4. 2023-08-10T06:30:59,044 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:table2,,1691649058228.8adb09ea4b151f0bc51e6dbcbab134a4. 2023-08-10T06:30:59,044 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:table2,,1691649058228.8adb09ea4b151f0bc51e6dbcbab134a4. after waiting 0 ms 2023-08-10T06:30:59,044 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:table2,,1691649058228.8adb09ea4b151f0bc51e6dbcbab134a4. 2023-08-10T06:30:59,044 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1909): Closed ns2:table2,,1691649058228.8adb09ea4b151f0bc51e6dbcbab134a4. 2023-08-10T06:30:59,044 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1622): Region close journal for 8adb09ea4b151f0bc51e6dbcbab134a4: 2023-08-10T06:30:59,045 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=139, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_ADD_TO_META 2023-08-10T06:30:59,045 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:table2,,1691649058228.8adb09ea4b151f0bc51e6dbcbab134a4.","families":{"info":[{"qualifier":"regioninfo","vlen":36,"tag":[],"timestamp":"1691649059045"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691649059045"}]},"ts":"1691649059045"} 2023-08-10T06:30:59,046 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:30:59,046 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=139, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-08-10T06:30:59,047 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649059047"}]},"ts":"1691649059047"} 2023-08-10T06:30:59,047 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:table2, state=ENABLING in hbase:meta 2023-08-10T06:30:59,056 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:table2, region=8adb09ea4b151f0bc51e6dbcbab134a4, ASSIGN}] 2023-08-10T06:30:59,058 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=140, ppid=139, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:table2, region=8adb09ea4b151f0bc51e6dbcbab134a4, ASSIGN 2023-08-10T06:30:59,058 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=140, ppid=139, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:table2, region=8adb09ea4b151f0bc51e6dbcbab134a4, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45101,1691648859755; forceNewPlan=false, retain=false 2023-08-10T06:30:59,210 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=140 updating hbase:meta row=8adb09ea4b151f0bc51e6dbcbab134a4, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:30:59,211 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE; OpenRegionProcedure 8adb09ea4b151f0bc51e6dbcbab134a4, server=jenkins-hbase3.apache.org,45101,1691648859755}] 2023-08-10T06:30:59,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=139 2023-08-10T06:30:59,369 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] handler.AssignRegionHandler(130): Open ns2:table2,,1691649058228.8adb09ea4b151f0bc51e6dbcbab134a4. 2023-08-10T06:30:59,369 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7254): Opening region: {ENCODED => 8adb09ea4b151f0bc51e6dbcbab134a4, NAME => 'ns2:table2,,1691649058228.8adb09ea4b151f0bc51e6dbcbab134a4.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:30:59,369 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:30:59,369 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table2 8adb09ea4b151f0bc51e6dbcbab134a4 2023-08-10T06:30:59,369 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(888): Instantiated ns2:table2,,1691649058228.8adb09ea4b151f0bc51e6dbcbab134a4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:30:59,369 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7296): checking encryption for 8adb09ea4b151f0bc51e6dbcbab134a4 2023-08-10T06:30:59,369 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7299): checking classloading for 8adb09ea4b151f0bc51e6dbcbab134a4 2023-08-10T06:30:59,371 INFO [StoreOpener-8adb09ea4b151f0bc51e6dbcbab134a4-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 8adb09ea4b151f0bc51e6dbcbab134a4 2023-08-10T06:30:59,372 INFO [StoreOpener-8adb09ea4b151f0bc51e6dbcbab134a4-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8adb09ea4b151f0bc51e6dbcbab134a4 columnFamilyName f 2023-08-10T06:30:59,372 DEBUG [StoreOpener-8adb09ea4b151f0bc51e6dbcbab134a4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:30:59,372 INFO [StoreOpener-8adb09ea4b151f0bc51e6dbcbab134a4-1 {}] regionserver.HStore(324): Store=8adb09ea4b151f0bc51e6dbcbab134a4/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:30:59,373 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/table2/8adb09ea4b151f0bc51e6dbcbab134a4 2023-08-10T06:30:59,373 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/table2/8adb09ea4b151f0bc51e6dbcbab134a4 2023-08-10T06:30:59,375 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1076): writing seq id for 8adb09ea4b151f0bc51e6dbcbab134a4 2023-08-10T06:30:59,377 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/table2/8adb09ea4b151f0bc51e6dbcbab134a4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:30:59,378 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1093): Opened 8adb09ea4b151f0bc51e6dbcbab134a4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9586819840, jitterRate=-0.10715782642364502}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-08-10T06:30:59,378 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(992): Region open journal for 8adb09ea4b151f0bc51e6dbcbab134a4: 2023-08-10T06:30:59,379 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegionServer(2552): Post open deploy tasks for ns2:table2,,1691649058228.8adb09ea4b151f0bc51e6dbcbab134a4., pid=141, masterSystemTime=1691649059363 2023-08-10T06:30:59,380 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegionServer(2579): Finished post open deploy task for ns2:table2,,1691649058228.8adb09ea4b151f0bc51e6dbcbab134a4. 2023-08-10T06:30:59,380 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] handler.AssignRegionHandler(158): Opened ns2:table2,,1691649058228.8adb09ea4b151f0bc51e6dbcbab134a4. 2023-08-10T06:30:59,380 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=140 updating hbase:meta row=8adb09ea4b151f0bc51e6dbcbab134a4, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:30:59,382 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=141, resume processing ppid=140 2023-08-10T06:30:59,382 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=141, ppid=140, state=SUCCESS; OpenRegionProcedure 8adb09ea4b151f0bc51e6dbcbab134a4, server=jenkins-hbase3.apache.org,45101,1691648859755 in 170 msec 2023-08-10T06:30:59,384 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=140, resume processing ppid=139 2023-08-10T06:30:59,384 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=140, ppid=139, state=SUCCESS; TransitRegionStateProcedure table=ns2:table2, region=8adb09ea4b151f0bc51e6dbcbab134a4, ASSIGN in 326 msec 2023-08-10T06:30:59,384 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=139, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-08-10T06:30:59,384 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649059384"}]},"ts":"1691649059384"} 2023-08-10T06:30:59,385 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:table2, state=ENABLED in hbase:meta 2023-08-10T06:30:59,387 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=139, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_POST_OPERATION 2023-08-10T06:30:59,388 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=139, state=SUCCESS; CreateTableProcedure table=ns2:table2 in 1.1580 sec 2023-08-10T06:31:00,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=139 2023-08-10T06:31:00,343 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:45379/backupUT/backup_1691649041585/ns2/test-16916488687341/archive/data/ns2/test-16916488687341/25a1c5b32e608be67dca00df059e0053 from hbase tables ns2:test-16916488687341 to tables ns2:table2 2023-08-10T06:31:00,343 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceRestoreJob(72): Restore ns2:test-16916488687341 into ns2:table2 2023-08-10T06:31:00,345 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-ns2-table2-1691649060343 from hdfs://localhost:45379/backupUT/backup_1691649041585/ns2/test-16916488687341/archive/data/ns2/test-16916488687341/25a1c5b32e608be67dca00df059e0053 2023-08-10T06:31:00,349 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@286ab18f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:31:00,350 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:31:00,351 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:52522, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:31:00,352 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-08-10T06:31:00,352 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:table2 2023-08-10T06:31:00,355 DEBUG [hconnection-0x4f57bfc9-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:31:00,356 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:36218, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:31:00,356 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:table2: 2023-08-10T06:31:00,357 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-08-10T06:31:00,357 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/partitions_326f1eb7-d5ea-48a6-bb81-4e90cd0b9b64 2023-08-10T06:31:00,557 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:table2' 2023-08-10T06:31:00,763 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:00,763 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:00,763 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:00,764 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:01,720 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/hadoop-8100219349345484915.jar 2023-08-10T06:31:01,720 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:01,721 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:01,721 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:01,721 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:01,722 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:01,722 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:01,722 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:01,723 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:01,723 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:01,724 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:31:01,724 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-08-10T06:31:01,725 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-08-10T06:31:01,725 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-08-10T06:31:01,725 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-08-10T06:31:01,726 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-08-10T06:31:01,726 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-08-10T06:31:01,726 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-08-10T06:31:01,727 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-08-10T06:31:01,727 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-08-10T06:31:01,728 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-08-10T06:31:01,728 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-08-10T06:31:01,728 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:01,729 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:01,729 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:01,729 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:01,729 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:01,730 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:01,730 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:31:01,730 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: ns2:table2 2023-08-10T06:31:01,731 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:31:01,731 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-08-10T06:31:01,731 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-08-10T06:31:01,732 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:31:01,742 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-08-10T06:31:01,815 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691649061783/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-08-10T06:31:01,815 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691649061783/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-08-10T06:31:01,862 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:45379/backupUT/backup_1691649041585/ns2/test-16916488687341/archive/data/ns2/test-16916488687341/25a1c5b32e608be67dca00df059e0053/f/06cc12c6a1d04d99aa771c07eb41201c_SeqId_4_ 2023-08-10T06:31:01,865 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-08-10T06:31:01,867 DEBUG [LocalJobRunner Map Task Executor #0 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 06cc12c6a1d04d99aa771c07eb41201c_SeqId_4_ 2023-08-10T06:31:01,885 DEBUG [pool-482-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6577b851, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:31:01,886 DEBUG [hconnection-0x7e2f572c-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:31:01,888 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:36228, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:31:01,891 DEBUG [pool-482-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:31:01,891 DEBUG [pool-482-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-08-10T06:31:01,891 DEBUG [pool-482-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-08-10T06:31:01,895 DEBUG [pool-482-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-ns2-table2-1691649060343/_temporary/0/_temporary/attempt_local726712189_0022_r_000000_0/f/bb9a7b6a7f6d4dfbbd9143096c8e8655 is 37, key is row10/f:q1/1691648873613/Put/seqid=0 2023-08-10T06:31:01,896 WARN [Thread-2320 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:45101] Specified favored nodes: [jenkins-hbase3.apache.org:45101] 2023-08-10T06:31:02,867 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-ns2-table2-1691649060343 2023-08-10T06:31:02,875 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2125c157, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:31:02,876 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:31:02,878 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:36240, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:31:02,884 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-ns2-table2-1691649060343/_SUCCESS 2023-08-10T06:31:02,888 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:31:02,890 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:52536, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:31:02,898 DEBUG [LoadIncrementalHFiles-0 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: bb9a7b6a7f6d4dfbbd9143096c8e8655 2023-08-10T06:31:02,898 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-ns2-table2-1691649060343/f/bb9a7b6a7f6d4dfbbd9143096c8e8655 first=Optional[row0] last=Optional[row98] 2023-08-10T06:31:02,899 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=ns2:table2,,1691649058228.8adb09ea4b151f0bc51e6dbcbab134a4., hostname=jenkins-hbase3.apache.org,45101,1691648859755, seqNum=2 for row with hfile group [{f,hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-ns2-table2-1691649060343/f/bb9a7b6a7f6d4dfbbd9143096c8e8655}] 2023-08-10T06:31:02,906 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-ns2-table2-1691649060343/f/bb9a7b6a7f6d4dfbbd9143096c8e8655 for inclusion in 8adb09ea4b151f0bc51e6dbcbab134a4/f 2023-08-10T06:31:02,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-08-10T06:31:02,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HStore(625): Region bounds: first= last= 2023-08-10T06:31:02,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: bb9a7b6a7f6d4dfbbd9143096c8e8655 2023-08-10T06:31:02,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HRegion(2520): Flush status journal for 8adb09ea4b151f0bc51e6dbcbab134a4: 2023-08-10T06:31:02,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-ns2-table2-1691649060343/f/bb9a7b6a7f6d4dfbbd9143096c8e8655 to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/staging/jenkins__ns2_table2__78aoa9imk9u99taijoh92lgvg1lanbevibhv7p13hvmfr99l81elguv4vundi3sn/f/bb9a7b6a7f6d4dfbbd9143096c8e8655 2023-08-10T06:31:02,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/staging/jenkins__ns2_table2__78aoa9imk9u99taijoh92lgvg1lanbevibhv7p13hvmfr99l81elguv4vundi3sn/f/bb9a7b6a7f6d4dfbbd9143096c8e8655 as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/table2/8adb09ea4b151f0bc51e6dbcbab134a4/f/bf268c111d8242b7a972c1a10cef756d_SeqId_4_ 2023-08-10T06:31:02,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@164987a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:31:02,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:31:02,920 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:52552, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:31:02,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:31:02,924 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:36252, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:31:02,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:31:02,936 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/staging/jenkins__ns2_table2__78aoa9imk9u99taijoh92lgvg1lanbevibhv7p13hvmfr99l81elguv4vundi3sn/f/bb9a7b6a7f6d4dfbbd9143096c8e8655 into 8adb09ea4b151f0bc51e6dbcbab134a4/f as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/table2/8adb09ea4b151f0bc51e6dbcbab134a4/f/bf268c111d8242b7a972c1a10cef756d_SeqId_4_ - updating store file list. 2023-08-10T06:31:02,940 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/table2/8adb09ea4b151f0bc51e6dbcbab134a4/f/bf268c111d8242b7a972c1a10cef756d_SeqId_4_ into 8adb09ea4b151f0bc51e6dbcbab134a4/f 2023-08-10T06:31:02,940 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/staging/jenkins__ns2_table2__78aoa9imk9u99taijoh92lgvg1lanbevibhv7p13hvmfr99l81elguv4vundi3sn/f/bb9a7b6a7f6d4dfbbd9143096c8e8655 into 8adb09ea4b151f0bc51e6dbcbab134a4/f (new location: hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/table2/8adb09ea4b151f0bc51e6dbcbab134a4/f/bf268c111d8242b7a972c1a10cef756d_SeqId_4_) 2023-08-10T06:31:02,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/staging/jenkins__ns2_table2__78aoa9imk9u99taijoh92lgvg1lanbevibhv7p13hvmfr99l81elguv4vundi3sn/f/bb9a7b6a7f6d4dfbbd9143096c8e8655 2023-08-10T06:31:02,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76c82795, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:31:02,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:31:02,948 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:52564, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-08-10T06:31:02,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:31:02,952 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:36268, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-08-10T06:31:02,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:31:02,964 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:31:02,964 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-08-10T06:31:02,964 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-08-10T06:31:02,964 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(233): Backup: backup_1691649041585 hdfs://localhost:45379/backupUT/backup_1691649041585/ns2/test-16916488687341/ 2023-08-10T06:31:02,965 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:45379/backupUT/backup_1691649041585/.backup.manifest 2023-08-10T06:31:02,968 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649041585/.backup.manifest 2023-08-10T06:31:02,968 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(148): Restoring 'ns3:test-16916488687342' to 'ns3:table3_restore' from full backup image hdfs://localhost:45379/backupUT/backup_1691649041585/ns3/test-16916488687342 2023-08-10T06:31:02,973 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] util.RestoreTool(101): Folder tableArchivePath: hdfs://localhost:45379/backupUT/backup_1691649041585/ns3/test-16916488687342/archive/data/ns3/test-16916488687342 does not exists 2023-08-10T06:31:02,973 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] util.RestoreTool(326): find table descriptor but no archive dir for table ns3:test-16916488687342, will only create table 2023-08-10T06:31:02,975 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] util.RestoreTool(487): Creating target table 'ns3:table3_restore' 2023-08-10T06:31:02,976 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'ns3:table3_restore', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:31:02,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns3:table3_restore 2023-08-10T06:31:02,978 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_PRE_OPERATION 2023-08-10T06:31:02,978 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns3" qualifier: "table3_restore" procId is: 142 2023-08-10T06:31:02,979 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=142, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-08-10T06:31:02,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=142 2023-08-10T06:31:03,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=142 2023-08-10T06:31:03,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=142 2023-08-10T06:31:03,386 INFO [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 4b60b5e5a6733a9997a44b4f0aed5224, NAME => 'ns3:table3_restore,,1691649062976.4b60b5e5a6733a9997a44b4f0aed5224.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:table3_restore', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e 2023-08-10T06:31:03,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=142 2023-08-10T06:31:03,791 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:table3_restore,,1691649062976.4b60b5e5a6733a9997a44b4f0aed5224.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:31:03,791 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1668): Closing 4b60b5e5a6733a9997a44b4f0aed5224, disabling compactions & flushes 2023-08-10T06:31:03,791 INFO [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:table3_restore,,1691649062976.4b60b5e5a6733a9997a44b4f0aed5224. 2023-08-10T06:31:03,791 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:table3_restore,,1691649062976.4b60b5e5a6733a9997a44b4f0aed5224. 2023-08-10T06:31:03,791 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:table3_restore,,1691649062976.4b60b5e5a6733a9997a44b4f0aed5224. after waiting 0 ms 2023-08-10T06:31:03,791 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:table3_restore,,1691649062976.4b60b5e5a6733a9997a44b4f0aed5224. 2023-08-10T06:31:03,791 INFO [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1909): Closed ns3:table3_restore,,1691649062976.4b60b5e5a6733a9997a44b4f0aed5224. 2023-08-10T06:31:03,791 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1622): Region close journal for 4b60b5e5a6733a9997a44b4f0aed5224: 2023-08-10T06:31:03,792 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=142, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_ADD_TO_META 2023-08-10T06:31:03,793 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:table3_restore,,1691649062976.4b60b5e5a6733a9997a44b4f0aed5224.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1691649063793"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691649063793"}]},"ts":"1691649063793"} 2023-08-10T06:31:03,794 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:31:03,795 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=142, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-08-10T06:31:03,795 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649063795"}]},"ts":"1691649063795"} 2023-08-10T06:31:03,796 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:table3_restore, state=ENABLING in hbase:meta 2023-08-10T06:31:03,801 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:table3_restore, region=4b60b5e5a6733a9997a44b4f0aed5224, ASSIGN}] 2023-08-10T06:31:03,802 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:table3_restore, region=4b60b5e5a6733a9997a44b4f0aed5224, ASSIGN 2023-08-10T06:31:03,803 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:table3_restore, region=4b60b5e5a6733a9997a44b4f0aed5224, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45101,1691648859755; forceNewPlan=false, retain=false 2023-08-10T06:31:03,955 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=143 updating hbase:meta row=4b60b5e5a6733a9997a44b4f0aed5224, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:31:03,957 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE; OpenRegionProcedure 4b60b5e5a6733a9997a44b4f0aed5224, server=jenkins-hbase3.apache.org,45101,1691648859755}] 2023-08-10T06:31:04,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=142 2023-08-10T06:31:04,111 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(130): Open ns3:table3_restore,,1691649062976.4b60b5e5a6733a9997a44b4f0aed5224. 2023-08-10T06:31:04,111 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7254): Opening region: {ENCODED => 4b60b5e5a6733a9997a44b4f0aed5224, NAME => 'ns3:table3_restore,,1691649062976.4b60b5e5a6733a9997a44b4f0aed5224.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:31:04,111 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:31:04,112 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table3_restore 4b60b5e5a6733a9997a44b4f0aed5224 2023-08-10T06:31:04,112 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(888): Instantiated ns3:table3_restore,,1691649062976.4b60b5e5a6733a9997a44b4f0aed5224.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:31:04,112 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7296): checking encryption for 4b60b5e5a6733a9997a44b4f0aed5224 2023-08-10T06:31:04,112 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7299): checking classloading for 4b60b5e5a6733a9997a44b4f0aed5224 2023-08-10T06:31:04,113 INFO [StoreOpener-4b60b5e5a6733a9997a44b4f0aed5224-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 4b60b5e5a6733a9997a44b4f0aed5224 2023-08-10T06:31:04,114 INFO [StoreOpener-4b60b5e5a6733a9997a44b4f0aed5224-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b60b5e5a6733a9997a44b4f0aed5224 columnFamilyName f 2023-08-10T06:31:04,114 DEBUG [StoreOpener-4b60b5e5a6733a9997a44b4f0aed5224-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:31:04,115 INFO [StoreOpener-4b60b5e5a6733a9997a44b4f0aed5224-1 {}] regionserver.HStore(324): Store=4b60b5e5a6733a9997a44b4f0aed5224/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:31:04,115 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns3/table3_restore/4b60b5e5a6733a9997a44b4f0aed5224 2023-08-10T06:31:04,115 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns3/table3_restore/4b60b5e5a6733a9997a44b4f0aed5224 2023-08-10T06:31:04,117 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1076): writing seq id for 4b60b5e5a6733a9997a44b4f0aed5224 2023-08-10T06:31:04,119 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns3/table3_restore/4b60b5e5a6733a9997a44b4f0aed5224/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:31:04,120 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1093): Opened 4b60b5e5a6733a9997a44b4f0aed5224; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11336591200, jitterRate=0.05580233037471771}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-08-10T06:31:04,120 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(992): Region open journal for 4b60b5e5a6733a9997a44b4f0aed5224: 2023-08-10T06:31:04,121 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2552): Post open deploy tasks for ns3:table3_restore,,1691649062976.4b60b5e5a6733a9997a44b4f0aed5224., pid=144, masterSystemTime=1691649064108 2023-08-10T06:31:04,122 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2579): Finished post open deploy task for ns3:table3_restore,,1691649062976.4b60b5e5a6733a9997a44b4f0aed5224. 2023-08-10T06:31:04,122 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(158): Opened ns3:table3_restore,,1691649062976.4b60b5e5a6733a9997a44b4f0aed5224. 2023-08-10T06:31:04,122 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=143 updating hbase:meta row=4b60b5e5a6733a9997a44b4f0aed5224, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:31:04,125 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=144, resume processing ppid=143 2023-08-10T06:31:04,125 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=144, ppid=143, state=SUCCESS; OpenRegionProcedure 4b60b5e5a6733a9997a44b4f0aed5224, server=jenkins-hbase3.apache.org,45101,1691648859755 in 168 msec 2023-08-10T06:31:04,126 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=143, resume processing ppid=142 2023-08-10T06:31:04,126 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=143, ppid=142, state=SUCCESS; TransitRegionStateProcedure table=ns3:table3_restore, region=4b60b5e5a6733a9997a44b4f0aed5224, ASSIGN in 325 msec 2023-08-10T06:31:04,127 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=142, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-08-10T06:31:04,127 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649064127"}]},"ts":"1691649064127"} 2023-08-10T06:31:04,128 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:table3_restore, state=ENABLED in hbase:meta 2023-08-10T06:31:04,131 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=142, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_POST_OPERATION 2023-08-10T06:31:04,132 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=142, state=SUCCESS; CreateTableProcedure table=ns3:table3_restore in 1.1550 sec 2023-08-10T06:31:04,912 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-08-10T06:31:05,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=142 2023-08-10T06:31:05,087 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-08-10T06:31:05,087 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(233): Backup: backup_1691649041585 hdfs://localhost:45379/backupUT/backup_1691649041585/ns3/test-16916488687342/ 2023-08-10T06:31:05,087 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-08-10T06:31:05,087 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:31:05,089 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster$12(2663): Client=jenkins//172.31.12.81 disable ns2:table2 2023-08-10T06:31:05,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=145, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=ns2:table2 2023-08-10T06:31:05,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=145 2023-08-10T06:31:05,092 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649065092"}]},"ts":"1691649065092"} 2023-08-10T06:31:05,093 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:table2, state=DISABLING in hbase:meta 2023-08-10T06:31:05,095 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(293): Set ns2:table2 to state=DISABLING 2023-08-10T06:31:05,095 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns2:table2, region=8adb09ea4b151f0bc51e6dbcbab134a4, UNASSIGN}] 2023-08-10T06:31:05,097 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=146, ppid=145, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns2:table2, region=8adb09ea4b151f0bc51e6dbcbab134a4, UNASSIGN 2023-08-10T06:31:05,097 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=146 updating hbase:meta row=8adb09ea4b151f0bc51e6dbcbab134a4, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:31:05,098 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE; CloseRegionProcedure 8adb09ea4b151f0bc51e6dbcbab134a4, server=jenkins-hbase3.apache.org,45101,1691648859755}] 2023-08-10T06:31:05,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=145 2023-08-10T06:31:05,250 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] handler.UnassignRegionHandler(119): Close 8adb09ea4b151f0bc51e6dbcbab134a4 2023-08-10T06:31:05,250 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1668): Closing 8adb09ea4b151f0bc51e6dbcbab134a4, disabling compactions & flushes 2023-08-10T06:31:05,250 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1690): Closing region ns2:table2,,1691649058228.8adb09ea4b151f0bc51e6dbcbab134a4. 2023-08-10T06:31:05,250 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:table2,,1691649058228.8adb09ea4b151f0bc51e6dbcbab134a4. 2023-08-10T06:31:05,250 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1778): Acquired close lock on ns2:table2,,1691649058228.8adb09ea4b151f0bc51e6dbcbab134a4. after waiting 0 ms 2023-08-10T06:31:05,250 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1788): Updates disabled for region ns2:table2,,1691649058228.8adb09ea4b151f0bc51e6dbcbab134a4. 2023-08-10T06:31:05,251 DEBUG [StoreFileCloser-8adb09ea4b151f0bc51e6dbcbab134a4-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: bf268c111d8242b7a972c1a10cef756d_SeqId_4_ 2023-08-10T06:31:05,256 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/table2/8adb09ea4b151f0bc51e6dbcbab134a4/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2023-08-10T06:31:05,256 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:31:05,257 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1909): Closed ns2:table2,,1691649058228.8adb09ea4b151f0bc51e6dbcbab134a4. 2023-08-10T06:31:05,257 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1622): Region close journal for 8adb09ea4b151f0bc51e6dbcbab134a4: 2023-08-10T06:31:05,258 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] handler.UnassignRegionHandler(163): Closed 8adb09ea4b151f0bc51e6dbcbab134a4 2023-08-10T06:31:05,258 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=146 updating hbase:meta row=8adb09ea4b151f0bc51e6dbcbab134a4, regionState=CLOSED 2023-08-10T06:31:05,260 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=147, resume processing ppid=146 2023-08-10T06:31:05,261 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=147, ppid=146, state=SUCCESS; CloseRegionProcedure 8adb09ea4b151f0bc51e6dbcbab134a4, server=jenkins-hbase3.apache.org,45101,1691648859755 in 161 msec 2023-08-10T06:31:05,262 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=146, resume processing ppid=145 2023-08-10T06:31:05,262 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=146, ppid=145, state=SUCCESS; TransitRegionStateProcedure table=ns2:table2, region=8adb09ea4b151f0bc51e6dbcbab134a4, UNASSIGN in 166 msec 2023-08-10T06:31:05,262 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649065262"}]},"ts":"1691649065262"} 2023-08-10T06:31:05,263 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:table2, state=DISABLED in hbase:meta 2023-08-10T06:31:05,264 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(305): Set ns2:table2 to state=DISABLED 2023-08-10T06:31:05,266 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=145, state=SUCCESS; DisableTableProcedure table=ns2:table2 in 176 msec 2023-08-10T06:31:05,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=145 2023-08-10T06:31:05,395 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster$5(2447): Client=jenkins//172.31.12.81 delete ns2:table2 2023-08-10T06:31:05,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=148, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=ns2:table2 2023-08-10T06:31:05,398 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=148, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=ns2:table2 2023-08-10T06:31:05,398 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=148, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=ns2:table2 2023-08-10T06:31:05,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=148 2023-08-10T06:31:05,400 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/table2/8adb09ea4b151f0bc51e6dbcbab134a4 2023-08-10T06:31:05,402 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/table2/8adb09ea4b151f0bc51e6dbcbab134a4/f, FileablePath, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/table2/8adb09ea4b151f0bc51e6dbcbab134a4/recovered.edits] 2023-08-10T06:31:05,431 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/table2/8adb09ea4b151f0bc51e6dbcbab134a4/f/bf268c111d8242b7a972c1a10cef756d_SeqId_4_ to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/archive/data/ns2/table2/8adb09ea4b151f0bc51e6dbcbab134a4/f/bf268c111d8242b7a972c1a10cef756d_SeqId_4_ 2023-08-10T06:31:05,434 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/table2/8adb09ea4b151f0bc51e6dbcbab134a4/recovered.edits/6.seqid to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/archive/data/ns2/table2/8adb09ea4b151f0bc51e6dbcbab134a4/recovered.edits/6.seqid 2023-08-10T06:31:05,434 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/table2/8adb09ea4b151f0bc51e6dbcbab134a4 2023-08-10T06:31:05,434 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(293): Archived ns2:table2 regions 2023-08-10T06:31:05,436 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=148, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=ns2:table2 2023-08-10T06:31:05,438 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of ns2:table2 from hbase:meta 2023-08-10T06:31:05,439 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(387): Removing 'ns2:table2' descriptor. 2023-08-10T06:31:05,440 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=148, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=ns2:table2 2023-08-10T06:31:05,440 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(377): Removing 'ns2:table2' from region states. 2023-08-10T06:31:05,440 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns2:table2,,1691649058228.8adb09ea4b151f0bc51e6dbcbab134a4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1691649065440"}]},"ts":"9223372036854775807"} 2023-08-10T06:31:05,442 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-08-10T06:31:05,442 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 8adb09ea4b151f0bc51e6dbcbab134a4, NAME => 'ns2:table2,,1691649058228.8adb09ea4b151f0bc51e6dbcbab134a4.', STARTKEY => '', ENDKEY => ''}] 2023-08-10T06:31:05,442 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(381): Marking 'ns2:table2' as deleted. 2023-08-10T06:31:05,442 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1691649065442"}]},"ts":"9223372036854775807"} 2023-08-10T06:31:05,443 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1678): Deleted table ns2:table2 state from META 2023-08-10T06:31:05,446 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(127): Finished pid=148, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=ns2:table2 2023-08-10T06:31:05,447 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=148, state=SUCCESS; DeleteTableProcedure table=ns2:table2 in 51 msec 2023-08-10T06:31:05,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=148 2023-08-10T06:31:05,501 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster$12(2663): Client=jenkins//172.31.12.81 disable ns3:table3_restore 2023-08-10T06:31:05,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=149, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=ns3:table3_restore 2023-08-10T06:31:05,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=149 2023-08-10T06:31:05,504 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649065504"}]},"ts":"1691649065504"} 2023-08-10T06:31:05,505 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:table3_restore, state=DISABLING in hbase:meta 2023-08-10T06:31:05,507 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(293): Set ns3:table3_restore to state=DISABLING 2023-08-10T06:31:05,508 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns3:table3_restore, region=4b60b5e5a6733a9997a44b4f0aed5224, UNASSIGN}] 2023-08-10T06:31:05,509 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=150, ppid=149, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns3:table3_restore, region=4b60b5e5a6733a9997a44b4f0aed5224, UNASSIGN 2023-08-10T06:31:05,510 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=150 updating hbase:meta row=4b60b5e5a6733a9997a44b4f0aed5224, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:31:05,511 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE; CloseRegionProcedure 4b60b5e5a6733a9997a44b4f0aed5224, server=jenkins-hbase3.apache.org,45101,1691648859755}] 2023-08-10T06:31:05,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=149 2023-08-10T06:31:05,663 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] handler.UnassignRegionHandler(119): Close 4b60b5e5a6733a9997a44b4f0aed5224 2023-08-10T06:31:05,664 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(1668): Closing 4b60b5e5a6733a9997a44b4f0aed5224, disabling compactions & flushes 2023-08-10T06:31:05,664 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(1690): Closing region ns3:table3_restore,,1691649062976.4b60b5e5a6733a9997a44b4f0aed5224. 2023-08-10T06:31:05,664 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:table3_restore,,1691649062976.4b60b5e5a6733a9997a44b4f0aed5224. 2023-08-10T06:31:05,664 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(1778): Acquired close lock on ns3:table3_restore,,1691649062976.4b60b5e5a6733a9997a44b4f0aed5224. after waiting 0 ms 2023-08-10T06:31:05,664 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(1788): Updates disabled for region ns3:table3_restore,,1691649062976.4b60b5e5a6733a9997a44b4f0aed5224. 2023-08-10T06:31:05,668 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns3/table3_restore/4b60b5e5a6733a9997a44b4f0aed5224/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-08-10T06:31:05,668 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:31:05,669 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(1909): Closed ns3:table3_restore,,1691649062976.4b60b5e5a6733a9997a44b4f0aed5224. 2023-08-10T06:31:05,669 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(1622): Region close journal for 4b60b5e5a6733a9997a44b4f0aed5224: 2023-08-10T06:31:05,670 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] handler.UnassignRegionHandler(163): Closed 4b60b5e5a6733a9997a44b4f0aed5224 2023-08-10T06:31:05,671 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=150 updating hbase:meta row=4b60b5e5a6733a9997a44b4f0aed5224, regionState=CLOSED 2023-08-10T06:31:05,673 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=151, resume processing ppid=150 2023-08-10T06:31:05,673 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=151, ppid=150, state=SUCCESS; CloseRegionProcedure 4b60b5e5a6733a9997a44b4f0aed5224, server=jenkins-hbase3.apache.org,45101,1691648859755 in 161 msec 2023-08-10T06:31:05,674 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=150, resume processing ppid=149 2023-08-10T06:31:05,674 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=150, ppid=149, state=SUCCESS; TransitRegionStateProcedure table=ns3:table3_restore, region=4b60b5e5a6733a9997a44b4f0aed5224, UNASSIGN in 165 msec 2023-08-10T06:31:05,674 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649065674"}]},"ts":"1691649065674"} 2023-08-10T06:31:05,675 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:table3_restore, state=DISABLED in hbase:meta 2023-08-10T06:31:05,677 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(305): Set ns3:table3_restore to state=DISABLED 2023-08-10T06:31:05,678 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=149, state=SUCCESS; DisableTableProcedure table=ns3:table3_restore in 177 msec 2023-08-10T06:31:05,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=149 2023-08-10T06:31:05,807 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster$5(2447): Client=jenkins//172.31.12.81 delete ns3:table3_restore 2023-08-10T06:31:05,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=152, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=ns3:table3_restore 2023-08-10T06:31:05,810 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=152, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-08-10T06:31:05,810 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=152, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-08-10T06:31:05,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=152 2023-08-10T06:31:05,812 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns3/table3_restore/4b60b5e5a6733a9997a44b4f0aed5224 2023-08-10T06:31:05,813 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns3/table3_restore/4b60b5e5a6733a9997a44b4f0aed5224/f, FileablePath, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns3/table3_restore/4b60b5e5a6733a9997a44b4f0aed5224/recovered.edits] 2023-08-10T06:31:05,819 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns3/table3_restore/4b60b5e5a6733a9997a44b4f0aed5224/recovered.edits/4.seqid to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/archive/data/ns3/table3_restore/4b60b5e5a6733a9997a44b4f0aed5224/recovered.edits/4.seqid 2023-08-10T06:31:05,819 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns3/table3_restore/4b60b5e5a6733a9997a44b4f0aed5224 2023-08-10T06:31:05,819 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(293): Archived ns3:table3_restore regions 2023-08-10T06:31:05,822 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=152, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-08-10T06:31:05,823 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of ns3:table3_restore from hbase:meta 2023-08-10T06:31:05,825 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(387): Removing 'ns3:table3_restore' descriptor. 2023-08-10T06:31:05,826 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=152, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-08-10T06:31:05,826 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(377): Removing 'ns3:table3_restore' from region states. 2023-08-10T06:31:05,826 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns3:table3_restore,,1691649062976.4b60b5e5a6733a9997a44b4f0aed5224.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1691649065826"}]},"ts":"9223372036854775807"} 2023-08-10T06:31:05,827 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-08-10T06:31:05,828 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 4b60b5e5a6733a9997a44b4f0aed5224, NAME => 'ns3:table3_restore,,1691649062976.4b60b5e5a6733a9997a44b4f0aed5224.', STARTKEY => '', ENDKEY => ''}] 2023-08-10T06:31:05,828 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(381): Marking 'ns3:table3_restore' as deleted. 2023-08-10T06:31:05,828 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1691649065828"}]},"ts":"9223372036854775807"} 2023-08-10T06:31:05,829 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1678): Deleted table ns3:table3_restore state from META 2023-08-10T06:31:05,831 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(127): Finished pid=152, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-08-10T06:31:05,832 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=152, state=SUCCESS; DeleteTableProcedure table=ns3:table3_restore in 24 msec 2023-08-10T06:31:05,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=152 2023-08-10T06:31:05,928 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreMultipleCommand Thread=603 (was 606), OpenFileDescriptor=679 (was 680), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=126 (was 162), ProcessCount=171 (was 171), AvailableMemoryMB=1692 (was 1800) 2023-08-10T06:31:05,928 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(130): Thread=603 is superior to 500 2023-08-10T06:31:05,943 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreSingleDNE Thread=603, OpenFileDescriptor=679, MaxFileDescriptor=60000, SystemLoadAverage=126, ProcessCount=171, AvailableMemoryMB=1691 2023-08-10T06:31:05,943 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(130): Thread=603 is superior to 500 2023-08-10T06:31:05,943 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] backup.TestFullRestore(260): test restore fails on a single table that does not exist 2023-08-10T06:31:05,949 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f11393c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:31:05,951 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:31:05,953 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:36270, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:31:05,954 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:31:05,955 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:52578, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:31:05,964 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-08-10T06:31:05,969 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:31:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691649065969 (current time:1691649065969). 2023-08-10T06:31:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:31:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-08-10T06:31:05,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e83b704, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:31:05,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:31:05,982 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:36284, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:31:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:31:05,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:31:05,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=153, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:31:05,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 153 2023-08-10T06:31:05,985 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:31:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=153 2023-08-10T06:31:05,986 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:31:05,988 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:31:06,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=153 2023-08-10T06:31:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=153 2023-08-10T06:31:06,393 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:31:06,393 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72}] 2023-08-10T06:31:06,395 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=154, ppid=153, state=RUNNABLE; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:31:06,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45101 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=154 2023-08-10T06:31:06,546 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. 2023-08-10T06:31:06,546 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.HRegion(2819): Flushing b9c493f2c9fbef45de19e8268a534e72 2/2 column families, dataSize=1.50 KB heapSize=2.73 KB 2023-08-10T06:31:06,550 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/meta/184a27fc671b4e9085d56df41f1234f5 is 171, key is trslm:hdfs://localhost:45379/backupUT\x00ns2:test-16916488687341/meta:log-roll-map/1691649057348/Put/seqid=0 2023-08-10T06:31:06,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=153 2023-08-10T06:31:06,954 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=717 B at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/meta/184a27fc671b4e9085d56df41f1234f5 2023-08-10T06:31:06,960 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 184a27fc671b4e9085d56df41f1234f5 2023-08-10T06:31:06,962 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/session/8a873f56ce4e4af992eb65b5390f76c2 is 524, key is session:backup_1691649041585/session:context/1691649058177/Put/seqid=0 2023-08-10T06:31:07,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=153 2023-08-10T06:31:07,369 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=820 B at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/session/8a873f56ce4e4af992eb65b5390f76c2 2023-08-10T06:31:07,373 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 8a873f56ce4e4af992eb65b5390f76c2 2023-08-10T06:31:07,374 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/meta/184a27fc671b4e9085d56df41f1234f5 as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/184a27fc671b4e9085d56df41f1234f5 2023-08-10T06:31:07,378 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/184a27fc671b4e9085d56df41f1234f5, entries=6, sequenceid=116, filesize=5.8 K 2023-08-10T06:31:07,379 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/session/8a873f56ce4e4af992eb65b5390f76c2 as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/8a873f56ce4e4af992eb65b5390f76c2 2023-08-10T06:31:07,384 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/8a873f56ce4e4af992eb65b5390f76c2, entries=2, sequenceid=116, filesize=5.5 K 2023-08-10T06:31:07,385 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.HRegion(3022): Finished flush of dataSize ~1.50 KB/1537, heapSize ~2.70 KB/2760, currentSize=0 B/0 for b9c493f2c9fbef45de19e8268a534e72 in 838ms, sequenceid=116, compaction requested=true 2023-08-10T06:31:07,385 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.HRegion(2520): Flush status journal for b9c493f2c9fbef45de19e8268a534e72: 2023-08-10T06:31:07,385 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. for snapshot_backup_system completed. 2023-08-10T06:31:07,385 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72.' region-info for snapshot=snapshot_backup_system 2023-08-10T06:31:07,385 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:31:07,385 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/bc10682e7f36491d941e2fe62e2e4100, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/9d9a4feb1c4e44d99cc63e3f68878731, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/f04a99990b2c429eaf1096ef413be3b8, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/9e97dc38b66b476f952990b23fa0a3ca, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/930c578fe3f543e1b6308abfb31c7246, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/24713cefebe0499c812c6d209dea0fd6, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/0ae449898702485faa63081b0f3f028a, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/cec52c43e8824ebfbbc2cf17b7a9ec5f, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/8f5b1bc0de87458e96bd38212599dcbd, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/184a27fc671b4e9085d56df41f1234f5] hfiles 2023-08-10T06:31:07,385 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (1/10): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/bc10682e7f36491d941e2fe62e2e4100 for snapshot=snapshot_backup_system 2023-08-10T06:31:07,385 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (2/10): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/9d9a4feb1c4e44d99cc63e3f68878731 for snapshot=snapshot_backup_system 2023-08-10T06:31:07,385 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (3/10): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/f04a99990b2c429eaf1096ef413be3b8 for snapshot=snapshot_backup_system 2023-08-10T06:31:07,385 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (4/10): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/9e97dc38b66b476f952990b23fa0a3ca for snapshot=snapshot_backup_system 2023-08-10T06:31:07,385 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (5/10): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/930c578fe3f543e1b6308abfb31c7246 for snapshot=snapshot_backup_system 2023-08-10T06:31:07,385 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (6/10): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/24713cefebe0499c812c6d209dea0fd6 for snapshot=snapshot_backup_system 2023-08-10T06:31:07,385 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (7/10): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/0ae449898702485faa63081b0f3f028a for snapshot=snapshot_backup_system 2023-08-10T06:31:07,385 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (8/10): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/cec52c43e8824ebfbbc2cf17b7a9ec5f for snapshot=snapshot_backup_system 2023-08-10T06:31:07,385 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (9/10): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/8f5b1bc0de87458e96bd38212599dcbd for snapshot=snapshot_backup_system 2023-08-10T06:31:07,385 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (10/10): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/184a27fc671b4e9085d56df41f1234f5 for snapshot=snapshot_backup_system 2023-08-10T06:31:07,385 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/e88678d5b4b14eecabac817daf12539e, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/37561b9a8ae2406bb6d053c33aeff31c, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/883911c19cab473f8bf801f934c90bc1, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/0c7705a45c554d6a97fa37fac319ceff, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/398a15bfffb84acfadb1be252c2e133a, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/adceb6320b02451abd02beed6c69c000, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/68fd0cf6c3094cc7af1a5176829fd9a4, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/40f9fbdeac5b49dbb654e429c177708d, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/75226fb0202e42819224b5e5920a02f2, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/3702fe0322c94bcc85372b5eb9c99029, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/8a873f56ce4e4af992eb65b5390f76c2] hfiles 2023-08-10T06:31:07,385 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (1/11): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/e88678d5b4b14eecabac817daf12539e for snapshot=snapshot_backup_system 2023-08-10T06:31:07,385 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (2/11): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/37561b9a8ae2406bb6d053c33aeff31c for snapshot=snapshot_backup_system 2023-08-10T06:31:07,385 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (3/11): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/883911c19cab473f8bf801f934c90bc1 for snapshot=snapshot_backup_system 2023-08-10T06:31:07,385 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (4/11): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/0c7705a45c554d6a97fa37fac319ceff for snapshot=snapshot_backup_system 2023-08-10T06:31:07,385 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (5/11): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/398a15bfffb84acfadb1be252c2e133a for snapshot=snapshot_backup_system 2023-08-10T06:31:07,385 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (6/11): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/adceb6320b02451abd02beed6c69c000 for snapshot=snapshot_backup_system 2023-08-10T06:31:07,385 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (7/11): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/68fd0cf6c3094cc7af1a5176829fd9a4 for snapshot=snapshot_backup_system 2023-08-10T06:31:07,385 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (8/11): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/40f9fbdeac5b49dbb654e429c177708d for snapshot=snapshot_backup_system 2023-08-10T06:31:07,385 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (9/11): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/75226fb0202e42819224b5e5920a02f2 for snapshot=snapshot_backup_system 2023-08-10T06:31:07,385 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (10/11): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/3702fe0322c94bcc85372b5eb9c99029 for snapshot=snapshot_backup_system 2023-08-10T06:31:07,385 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (11/11): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/8a873f56ce4e4af992eb65b5390f76c2 for snapshot=snapshot_backup_system 2023-08-10T06:31:07,390 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. 2023-08-10T06:31:07,390 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=154 2023-08-10T06:31:07,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster(3994): Remote procedure done, pid=154 2023-08-10T06:31:07,391 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:31:07,392 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=154, ppid=153, state=RUNNABLE; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:31:07,394 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=154, resume processing ppid=153 2023-08-10T06:31:07,394 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=154, ppid=153, state=SUCCESS; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72 in 1.0000 sec 2023-08-10T06:31:07,394 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:31:07,395 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:31:07,395 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:31:07,395 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-08-10T06:31:07,396 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_backup_system 2023-08-10T06:31:07,810 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:31:07,823 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:31:07,823 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_backup_system 2023-08-10T06:31:08,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=153 2023-08-10T06:31:08,642 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:31:08,642 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 153 2023-08-10T06:31:08,643 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=153, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.6590 sec 2023-08-10T06:31:10,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=153 2023-08-10T06:31:10,093 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(115): Backup backup_1691649065949 started at 1691649070093. 2023-08-10T06:31:10,094 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(119): Backup session backup_1691649065949 has been started. 2023-08-10T06:31:10,101 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-08-10T06:31:10,101 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-08-10T06:31:10,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-08-10T06:31:10,102 INFO [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-08-10T06:31:10,102 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-08-10T06:31:10,102 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-08-10T06:31:10,102 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-08-10T06:31:10,103 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:31:10,104 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:31:10,104 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:31:10,104 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:31:10,105 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-08-10T06:31:10,105 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-08-10T06:31:10,105 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-08-10T06:31:10,105 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:31:10,105 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-08-10T06:31:10,105 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-08-10T06:31:10,106 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-08-10T06:31:10,106 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-08-10T06:31:10,106 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-08-10T06:31:10,106 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-08-10T06:31:10,106 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-08-10T06:31:10,106 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,45101,1691648859755' joining acquired barrier for procedure (rolllog) in zk 2023-08-10T06:31:10,108 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-08-10T06:31:10,108 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:31:10,108 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:31:10,108 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:31:10,109 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-08-10T06:31:10,109 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:31:10,109 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:31:10,109 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:31:10,109 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:31:10,110 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:31:10,110 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,45101,1691648859755' joining acquired barrier for procedure 'rolllog' on coordinator 2023-08-10T06:31:10,110 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@2ab39083[Count = 0] remaining members to acquire global barrier 2023-08-10T06:31:10,110 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-08-10T06:31:10,110 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-08-10T06:31:10,112 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-08-10T06:31:10,112 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-08-10T06:31:10,112 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-08-10T06:31:10,112 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-08-10T06:31:10,112 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-08-10T06:31:10,112 DEBUG [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:31:10,112 INFO [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1691649045762 highest: 1691649045762 on jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:31:10,112 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta:.meta(num 1691649045746) roll requested 2023-08-10T06:31:10,124 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691649070113.meta, exclude list is [], retry=0 2023-08-10T06:31:10,127 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK] 2023-08-10T06:31:10,128 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691649045746.meta with entries=24, filesize=5.77 KB; new WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691649070113.meta 2023-08-10T06:31:10,128 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK]] 2023-08-10T06:31:10,128 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691649045746.meta is not closed yet, will try archiving it next time 2023-08-10T06:31:10,128 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C45101%2C1691648859755:(num 1691649045762) roll requested 2023-08-10T06:31:10,139 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691649070128, exclude list is [], retry=0 2023-08-10T06:31:10,142 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK] 2023-08-10T06:31:10,143 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691649045762 with entries=15, filesize=4.05 KB; new WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691649070128 2023-08-10T06:31:10,143 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK]] 2023-08-10T06:31:10,143 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691649045762 is not closed yet, will try archiving it next time 2023-08-10T06:31:10,153 DEBUG [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 41 2023-08-10T06:31:10,153 INFO [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1691649070128 on jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:31:10,159 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-08-10T06:31:10,159 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,45101,1691648859755' in zk 2023-08-10T06:31:10,161 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-08-10T06:31:10,161 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:31:10,161 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:31:10,161 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-08-10T06:31:10,161 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:31:10,162 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:31:10,162 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:31:10,162 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:31:10,163 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:31:10,163 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:31:10,163 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:31:10,163 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:31:10,164 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:31:10,164 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,45101,1691648859755': 2023-08-10T06:31:10,164 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,45101,1691648859755' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-08-10T06:31:10,164 INFO [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-08-10T06:31:10,164 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-08-10T06:31:10,164 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-08-10T06:31:10,164 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-08-10T06:31:10,164 INFO [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-08-10T06:31:10,166 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-08-10T06:31:10,166 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:31:10,166 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-08-10T06:31:10,166 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:31:10,166 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:31:10,166 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:31:10,166 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:31:10,166 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:31:10,166 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:31:10,167 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:31:10,167 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:31:10,167 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:31:10,168 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:31:10,168 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:31:10,168 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:31:10,168 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:31:10,171 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:31:10,171 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:31:10,171 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:31:10,171 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-08-10T06:31:10,171 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-08-10T06:31:10,172 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:31:10,172 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:31:10,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-08-10T06:31:10,276 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1691649070275_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } 2023-08-10T06:31:10,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691649070276 (current time:1691649070276). 2023-08-10T06:31:10,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:31:10,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1691649070275_default_test-1691648868734 VERSION not specified, setting to 2 2023-08-10T06:31:10,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@176fd979, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:31:10,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:31:10,285 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:33906, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:31:10,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:31:10,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:31:10,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=155, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1691649070275_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } 2023-08-10T06:31:10,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1691649070275_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 }, snapshot procedure id = 155 2023-08-10T06:31:10,288 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1691649070275_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:31:10,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=155 2023-08-10T06:31:10,288 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1691649070275_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:31:10,291 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1691649070275_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:31:10,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=155 2023-08-10T06:31:10,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=155 2023-08-10T06:31:10,696 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1691649070275_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:31:10,696 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=156, ppid=155, state=RUNNABLE; SnapshotRegionProcedure 513ad41c456db08f6287ccd38ca6b19c}] 2023-08-10T06:31:10,698 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=156, ppid=155, state=RUNNABLE; SnapshotRegionProcedure 513ad41c456db08f6287ccd38ca6b19c 2023-08-10T06:31:10,758 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-08-10T06:31:10,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45101 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=156 2023-08-10T06:31:10,849 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1691648868734,,1691648971834.513ad41c456db08f6287ccd38ca6b19c. 2023-08-10T06:31:10,850 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] regionserver.HRegion(2520): Flush status journal for 513ad41c456db08f6287ccd38ca6b19c: 2023-08-10T06:31:10,850 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1691648868734,,1691648971834.513ad41c456db08f6287ccd38ca6b19c. for snapshot_1691649070275_default_test-1691648868734 completed. 2023-08-10T06:31:10,850 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] snapshot.SnapshotManifest(240): Storing 'test-1691648868734,,1691648971834.513ad41c456db08f6287ccd38ca6b19c.' region-info for snapshot=snapshot_1691649070275_default_test-1691648868734 2023-08-10T06:31:10,850 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:31:10,850 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/513ad41c456db08f6287ccd38ca6b19c/f/8eec1fbc8346444d9f3bf9d53b64b2b2_SeqId_4_] hfiles 2023-08-10T06:31:10,850 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/513ad41c456db08f6287ccd38ca6b19c/f/8eec1fbc8346444d9f3bf9d53b64b2b2_SeqId_4_ for snapshot=snapshot_1691649070275_default_test-1691648868734 2023-08-10T06:31:10,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=155 2023-08-10T06:31:11,255 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1691648868734,,1691648971834.513ad41c456db08f6287ccd38ca6b19c. 2023-08-10T06:31:11,255 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=156 2023-08-10T06:31:11,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster(3994): Remote procedure done, pid=156 2023-08-10T06:31:11,256 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1691649070275_default_test-1691648868734 on region 513ad41c456db08f6287ccd38ca6b19c 2023-08-10T06:31:11,257 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=156, ppid=155, state=RUNNABLE; SnapshotRegionProcedure 513ad41c456db08f6287ccd38ca6b19c 2023-08-10T06:31:11,259 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=156, resume processing ppid=155 2023-08-10T06:31:11,259 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=156, ppid=155, state=SUCCESS; SnapshotRegionProcedure 513ad41c456db08f6287ccd38ca6b19c in 562 msec 2023-08-10T06:31:11,259 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1691649070275_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:31:11,259 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1691649070275_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:31:11,260 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1691649070275_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:31:11,260 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1691649070275_default_test-1691648868734 2023-08-10T06:31:11,260 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_1691649070275_default_test-1691648868734 2023-08-10T06:31:11,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=155 2023-08-10T06:31:11,668 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1691649070275_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:31:11,673 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1691649070275_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:31:11,674 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_1691649070275_default_test-1691648868734 to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691649070275_default_test-1691648868734 2023-08-10T06:31:12,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=155 2023-08-10T06:31:12,491 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1691649070275_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:31:12,491 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1691649070275_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 }, snapshot procedure id = 155 2023-08-10T06:31:12,492 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=155, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1691649070275_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } in 2.2050 sec 2023-08-10T06:31:14,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=155 2023-08-10T06:31:14,394 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1691649065949 2023-08-10T06:31:14,394 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-08-10T06:31:14,394 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-08-10T06:31:14,394 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1691649065949_test-1691648868734 2023-08-10T06:31:14,394 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1691649070275_default_test-1691648868734 to hdfs://localhost:45379/backupUT/backup_1691649065949/default/test-1691648868734/ 2023-08-10T06:31:14,394 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-08-10T06:31:14,416 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:45379, inputRoot=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e 2023-08-10T06:31:14,416 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,33705,1691648856595_1069938501_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:45379/backupUT/backup_1691649065949/default/test-1691648868734, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:45379/backupUT/backup_1691649065949/default/test-1691648868734/.hbase-snapshot/.tmp/snapshot_1691649070275_default_test-1691648868734 2023-08-10T06:31:14,416 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:45379, inputRoot=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e, snapshotDir=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691649070275_default_test-1691648868734. 2023-08-10T06:31:14,425 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691649070275_default_test-1691648868734 to hdfs://localhost:45379/backupUT/backup_1691649065949/default/test-1691648868734/.hbase-snapshot/.tmp/snapshot_1691649070275_default_test-1691648868734 2023-08-10T06:31:14,840 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:14,841 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:14,841 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:14,841 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:15,801 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/hadoop-8399179563261749923.jar 2023-08-10T06:31:15,802 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:15,802 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:15,802 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:15,803 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:15,803 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:15,803 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:15,804 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:15,804 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:15,804 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:15,805 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:31:15,806 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-08-10T06:31:15,806 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-08-10T06:31:15,806 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-08-10T06:31:15,807 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-08-10T06:31:15,807 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-08-10T06:31:15,807 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-08-10T06:31:15,808 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-08-10T06:31:15,808 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-08-10T06:31:15,808 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-08-10T06:31:15,809 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-08-10T06:31:15,809 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-08-10T06:31:15,810 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:31:15,810 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:31:15,811 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:31:15,811 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:31:15,811 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:31:15,812 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:31:15,812 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:31:15,853 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1691649070275_default_test-1691648868734' hfile list 2023-08-10T06:31:15,855 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-08-10T06:31:15,923 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691649075890/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-08-10T06:31:15,923 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691649075890/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-08-10T06:31:15,974 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-08-10T06:31:15,997 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/513ad41c456db08f6287ccd38ca6b19c/f/8eec1fbc8346444d9f3bf9d53b64b2b2_SeqId_4_ output=hdfs://localhost:45379/backupUT/backup_1691649065949/default/test-1691648868734/archive/data/default/test-1691648868734/513ad41c456db08f6287ccd38ca6b19c/f/8eec1fbc8346444d9f3bf9d53b64b2b2_SeqId_4_ 2023-08-10T06:31:15,997 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8724 (8.5 K) time=0sec 8.320M/sec 2023-08-10T06:31:16,957 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-08-10T06:31:16,958 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-08-10T06:31:16,963 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1691649070275_default_test-1691648868734 2023-08-10T06:31:16,963 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1691649070275_default_test-1691648868734 finished. 2023-08-10T06:31:16,963 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(1014): test-1691648868734 2023-08-10T06:31:16,969 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691649065949 2023-08-10T06:31:16,969 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-08-10T06:31:17,374 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:45379/backupUT/backup_1691649065949/.backup.manifest 2023-08-10T06:31:17,374 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(358): Backup backup_1691649065949 finished: type=FULL,tablelist=test-1691648868734,targetRootDir=hdfs://localhost:45379/backupUT,startts=1691649070093,completets=1691649076969,bytescopied=0 2023-08-10T06:31:17,374 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-08-10T06:31:17,374 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1691649070275_default_test-1691648868734 2023-08-10T06:31:17,375 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1691649070275_default_test-1691648868734" 2023-08-10T06:31:17,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1691649070275_default_test-1691648868734 2023-08-10T06:31:17,378 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1691649070275_default_test-1691648868734 for backup backup_1691649065949 succeeded. 2023-08-10T06:31:17,379 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-08-10T06:31:17,383 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-08-10T06:31:17,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-08-10T06:31:17,385 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-08-10T06:31:17,386 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-08-10T06:31:17,390 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(376): Backup backup_1691649065949 completed. 2023-08-10T06:31:17,390 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:31:17,400 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] backup.TestFullRestore(265): backup complete 2023-08-10T06:31:17,400 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(101): HBase table table1 does not exist. It will be created during restore process 2023-08-10T06:31:17,401 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:45379/backupUT/backup_1691649065949/.backup.manifest 2023-08-10T06:31:17,403 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649065949/.backup.manifest 2023-08-10T06:31:17,403 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:45379/backupUT/backup_1691649065949/.backup.manifest 2023-08-10T06:31:17,405 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649065949/.backup.manifest 2023-08-10T06:31:17,405 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(148): Restoring 'faketable' to 'table1' from full backup image hdfs://localhost:45379/backupUT/backup_1691649065949/default/faketable 2023-08-10T06:31:17,422 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreSingleDNE Thread=602 (was 603), OpenFileDescriptor=686 (was 679) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=106 (was 126), ProcessCount=171 (was 171), AvailableMemoryMB=1619 (was 1691) 2023-08-10T06:31:17,422 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(130): Thread=602 is superior to 500 2023-08-10T06:31:17,437 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreSingleCommand Thread=602, OpenFileDescriptor=686, MaxFileDescriptor=60000, SystemLoadAverage=106, ProcessCount=171, AvailableMemoryMB=1619 2023-08-10T06:31:17,437 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(130): Thread=602 is superior to 500 2023-08-10T06:31:17,437 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] backup.TestFullRestore(76): test full restore on a single table empty table: command-line 2023-08-10T06:31:17,443 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a89fe98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:31:17,447 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:31:17,448 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:33908, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:31:17,449 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:31:17,451 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:53412, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:31:17,461 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-08-10T06:31:17,468 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:31:17,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691649077468 (current time:1691649077468). 2023-08-10T06:31:17,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:31:17,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-08-10T06:31:17,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9b348f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:31:17,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:31:17,478 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:33922, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:31:17,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:31:17,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:31:17,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=157, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:31:17,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 157 2023-08-10T06:31:17,482 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:31:17,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=157 2023-08-10T06:31:17,483 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:31:17,485 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:31:17,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=157 2023-08-10T06:31:17,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=157 2023-08-10T06:31:17,891 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:31:17,891 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=158, ppid=157, state=RUNNABLE; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72}] 2023-08-10T06:31:17,893 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=158, ppid=157, state=RUNNABLE; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:31:18,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45101 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=158 2023-08-10T06:31:18,045 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. 2023-08-10T06:31:18,045 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.HRegion(2819): Flushing b9c493f2c9fbef45de19e8268a534e72 2/2 column families, dataSize=1.14 KB heapSize=2.22 KB 2023-08-10T06:31:18,048 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/meta/225d36cdae6a48eb86402cbd55c85ab2 is 169, key is trslm:hdfs://localhost:45379/backupUT\x00test-1691648868734/meta:log-roll-map/1691649076966/Put/seqid=0 2023-08-10T06:31:18,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=157 2023-08-10T06:31:18,453 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=451 B at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/meta/225d36cdae6a48eb86402cbd55c85ab2 2023-08-10T06:31:18,458 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 225d36cdae6a48eb86402cbd55c85ab2 2023-08-10T06:31:18,461 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/session/1dd3913d29eb48739f0c6f2120db1ded is 452, key is session:backup_1691649065949/session:context/1691649077385/Put/seqid=0 2023-08-10T06:31:18,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=157 2023-08-10T06:31:18,865 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=721 B at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/session/1dd3913d29eb48739f0c6f2120db1ded 2023-08-10T06:31:18,869 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 1dd3913d29eb48739f0c6f2120db1ded 2023-08-10T06:31:18,870 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/meta/225d36cdae6a48eb86402cbd55c85ab2 as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/225d36cdae6a48eb86402cbd55c85ab2 2023-08-10T06:31:18,874 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/225d36cdae6a48eb86402cbd55c85ab2, entries=4, sequenceid=127, filesize=5.5 K 2023-08-10T06:31:18,875 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/session/1dd3913d29eb48739f0c6f2120db1ded as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/1dd3913d29eb48739f0c6f2120db1ded 2023-08-10T06:31:18,879 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/1dd3913d29eb48739f0c6f2120db1ded, entries=2, sequenceid=127, filesize=5.4 K 2023-08-10T06:31:18,879 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1172, heapSize ~2.19 KB/2240, currentSize=0 B/0 for b9c493f2c9fbef45de19e8268a534e72 in 834ms, sequenceid=127, compaction requested=true 2023-08-10T06:31:18,879 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.HRegion(2520): Flush status journal for b9c493f2c9fbef45de19e8268a534e72: 2023-08-10T06:31:18,879 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. for snapshot_backup_system completed. 2023-08-10T06:31:18,880 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72.' region-info for snapshot=snapshot_backup_system 2023-08-10T06:31:18,880 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:31:18,880 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/bc10682e7f36491d941e2fe62e2e4100, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/9d9a4feb1c4e44d99cc63e3f68878731, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/f04a99990b2c429eaf1096ef413be3b8, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/9e97dc38b66b476f952990b23fa0a3ca, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/930c578fe3f543e1b6308abfb31c7246, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/24713cefebe0499c812c6d209dea0fd6, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/0ae449898702485faa63081b0f3f028a, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/cec52c43e8824ebfbbc2cf17b7a9ec5f, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/8f5b1bc0de87458e96bd38212599dcbd, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/184a27fc671b4e9085d56df41f1234f5, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/225d36cdae6a48eb86402cbd55c85ab2] hfiles 2023-08-10T06:31:18,880 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (1/11): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/bc10682e7f36491d941e2fe62e2e4100 for snapshot=snapshot_backup_system 2023-08-10T06:31:18,880 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (2/11): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/9d9a4feb1c4e44d99cc63e3f68878731 for snapshot=snapshot_backup_system 2023-08-10T06:31:18,880 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (3/11): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/f04a99990b2c429eaf1096ef413be3b8 for snapshot=snapshot_backup_system 2023-08-10T06:31:18,880 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (4/11): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/9e97dc38b66b476f952990b23fa0a3ca for snapshot=snapshot_backup_system 2023-08-10T06:31:18,880 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (5/11): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/930c578fe3f543e1b6308abfb31c7246 for snapshot=snapshot_backup_system 2023-08-10T06:31:18,880 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (6/11): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/24713cefebe0499c812c6d209dea0fd6 for snapshot=snapshot_backup_system 2023-08-10T06:31:18,880 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (7/11): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/0ae449898702485faa63081b0f3f028a for snapshot=snapshot_backup_system 2023-08-10T06:31:18,880 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (8/11): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/cec52c43e8824ebfbbc2cf17b7a9ec5f for snapshot=snapshot_backup_system 2023-08-10T06:31:18,880 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (9/11): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/8f5b1bc0de87458e96bd38212599dcbd for snapshot=snapshot_backup_system 2023-08-10T06:31:18,880 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (10/11): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/184a27fc671b4e9085d56df41f1234f5 for snapshot=snapshot_backup_system 2023-08-10T06:31:18,880 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (11/11): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/225d36cdae6a48eb86402cbd55c85ab2 for snapshot=snapshot_backup_system 2023-08-10T06:31:18,880 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/e88678d5b4b14eecabac817daf12539e, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/37561b9a8ae2406bb6d053c33aeff31c, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/883911c19cab473f8bf801f934c90bc1, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/0c7705a45c554d6a97fa37fac319ceff, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/398a15bfffb84acfadb1be252c2e133a, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/adceb6320b02451abd02beed6c69c000, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/68fd0cf6c3094cc7af1a5176829fd9a4, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/40f9fbdeac5b49dbb654e429c177708d, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/75226fb0202e42819224b5e5920a02f2, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/3702fe0322c94bcc85372b5eb9c99029, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/8a873f56ce4e4af992eb65b5390f76c2, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/1dd3913d29eb48739f0c6f2120db1ded] hfiles 2023-08-10T06:31:18,880 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (1/12): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/e88678d5b4b14eecabac817daf12539e for snapshot=snapshot_backup_system 2023-08-10T06:31:18,880 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (2/12): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/37561b9a8ae2406bb6d053c33aeff31c for snapshot=snapshot_backup_system 2023-08-10T06:31:18,880 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (3/12): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/883911c19cab473f8bf801f934c90bc1 for snapshot=snapshot_backup_system 2023-08-10T06:31:18,880 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (4/12): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/0c7705a45c554d6a97fa37fac319ceff for snapshot=snapshot_backup_system 2023-08-10T06:31:18,880 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (5/12): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/398a15bfffb84acfadb1be252c2e133a for snapshot=snapshot_backup_system 2023-08-10T06:31:18,880 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (6/12): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/adceb6320b02451abd02beed6c69c000 for snapshot=snapshot_backup_system 2023-08-10T06:31:18,880 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (7/12): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/68fd0cf6c3094cc7af1a5176829fd9a4 for snapshot=snapshot_backup_system 2023-08-10T06:31:18,880 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (8/12): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/40f9fbdeac5b49dbb654e429c177708d for snapshot=snapshot_backup_system 2023-08-10T06:31:18,880 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (9/12): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/75226fb0202e42819224b5e5920a02f2 for snapshot=snapshot_backup_system 2023-08-10T06:31:18,880 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (10/12): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/3702fe0322c94bcc85372b5eb9c99029 for snapshot=snapshot_backup_system 2023-08-10T06:31:18,880 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (11/12): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/8a873f56ce4e4af992eb65b5390f76c2 for snapshot=snapshot_backup_system 2023-08-10T06:31:18,880 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (12/12): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/1dd3913d29eb48739f0c6f2120db1ded for snapshot=snapshot_backup_system 2023-08-10T06:31:19,285 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. 2023-08-10T06:31:19,285 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=158 2023-08-10T06:31:19,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster(3994): Remote procedure done, pid=158 2023-08-10T06:31:19,285 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:31:19,287 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=158, ppid=157, state=RUNNABLE; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:31:19,288 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=158, resume processing ppid=157 2023-08-10T06:31:19,288 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=158, ppid=157, state=SUCCESS; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72 in 1.3960 sec 2023-08-10T06:31:19,288 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:31:19,289 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:31:19,289 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:31:19,289 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-08-10T06:31:19,291 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_backup_system 2023-08-10T06:31:19,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=157 2023-08-10T06:31:19,700 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:31:19,713 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:31:19,713 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_backup_system 2023-08-10T06:31:20,528 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:31:20,528 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 157 2023-08-10T06:31:20,529 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=157, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 3.0490 sec 2023-08-10T06:31:21,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=157 2023-08-10T06:31:21,588 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(115): Backup backup_1691649077443 started at 1691649081588. 2023-08-10T06:31:21,590 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(119): Backup session backup_1691649077443 has been started. 2023-08-10T06:31:21,594 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-08-10T06:31:21,594 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-08-10T06:31:21,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure.ProcedureCoordinator(143): Procedure rolllog was in running list but was completed. Accepting new attempt. 2023-08-10T06:31:21,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-08-10T06:31:21,594 INFO [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-08-10T06:31:21,595 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-08-10T06:31:21,596 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-08-10T06:31:21,596 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-08-10T06:31:21,596 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:31:21,597 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:31:21,597 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:31:21,597 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:31:21,597 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-08-10T06:31:21,598 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-08-10T06:31:21,598 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-08-10T06:31:21,598 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:31:21,598 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-08-10T06:31:21,598 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-08-10T06:31:21,598 WARN [zk-event-processor-pool-0 {}] procedure.ProcedureMember(133): A completed old subproc rolllog is still present, removing 2023-08-10T06:31:21,598 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-08-10T06:31:21,598 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-08-10T06:31:21,598 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-08-10T06:31:21,599 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-08-10T06:31:21,599 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-08-10T06:31:21,599 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,45101,1691648859755' joining acquired barrier for procedure (rolllog) in zk 2023-08-10T06:31:21,601 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-08-10T06:31:21,601 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:31:21,601 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:31:21,601 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:31:21,601 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-08-10T06:31:21,601 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:31:21,602 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:31:21,602 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:31:21,602 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:31:21,602 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:31:21,603 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,45101,1691648859755' joining acquired barrier for procedure 'rolllog' on coordinator 2023-08-10T06:31:21,603 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@4212a31[Count = 0] remaining members to acquire global barrier 2023-08-10T06:31:21,603 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-08-10T06:31:21,603 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-08-10T06:31:21,605 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-08-10T06:31:21,605 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-08-10T06:31:21,605 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-08-10T06:31:21,605 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-08-10T06:31:21,605 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-08-10T06:31:21,605 DEBUG [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:31:21,605 INFO [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1691649070128 highest: 1691649070128 on jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:31:21,605 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta:.meta(num 1691649070113) roll requested 2023-08-10T06:31:21,616 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691649081605.meta, exclude list is [], retry=0 2023-08-10T06:31:21,621 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK] 2023-08-10T06:31:21,622 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691649070113.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691649081605.meta 2023-08-10T06:31:21,622 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK]] 2023-08-10T06:31:21,622 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691649070113.meta is not closed yet, will try archiving it next time 2023-08-10T06:31:21,622 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C45101%2C1691648859755:(num 1691649070128) roll requested 2023-08-10T06:31:21,624 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691649070113.meta not finished, retry = 0 2023-08-10T06:31:21,633 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691649081622, exclude list is [], retry=0 2023-08-10T06:31:21,636 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK] 2023-08-10T06:31:21,637 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691649070128 with entries=10, filesize=2.31 KB; new WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691649081622 2023-08-10T06:31:21,637 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK]] 2023-08-10T06:31:21,637 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691649070128 is not closed yet, will try archiving it next time 2023-08-10T06:31:21,637 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691649045762 to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/oldWALs/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691649045762 2023-08-10T06:31:21,645 DEBUG [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-08-10T06:31:21,645 INFO [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1691649081622 on jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:31:21,652 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-08-10T06:31:21,652 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,45101,1691648859755' in zk 2023-08-10T06:31:21,654 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-08-10T06:31:21,654 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:31:21,654 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:31:21,654 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-08-10T06:31:21,654 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:31:21,655 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:31:21,655 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:31:21,655 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:31:21,655 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:31:21,656 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:31:21,656 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:31:21,656 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:31:21,656 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:31:21,657 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,45101,1691648859755': 2023-08-10T06:31:21,657 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,45101,1691648859755' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-08-10T06:31:21,657 INFO [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-08-10T06:31:21,657 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-08-10T06:31:21,657 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-08-10T06:31:21,657 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-08-10T06:31:21,657 INFO [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-08-10T06:31:21,658 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-08-10T06:31:21,658 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-08-10T06:31:21,658 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:31:21,658 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:31:21,658 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:31:21,659 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:31:21,659 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:31:21,659 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:31:21,659 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:31:21,659 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:31:21,659 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:31:21,660 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:31:21,660 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:31:21,660 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:31:21,660 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:31:21,661 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:31:21,668 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:31:21,668 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:31:21,668 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:31:21,668 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-08-10T06:31:21,669 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-08-10T06:31:21,669 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:31:21,669 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:31:21,725 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691649070113.meta to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/oldWALs/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691649070113.meta 2023-08-10T06:31:21,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-08-10T06:31:21,773 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1691649081772_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } 2023-08-10T06:31:21,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691649081773 (current time:1691649081773). 2023-08-10T06:31:21,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:31:21,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1691649081772_default_test-1691648868734 VERSION not specified, setting to 2 2023-08-10T06:31:21,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14094240, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:31:21,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:31:21,781 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:47526, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:31:21,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:31:21,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:31:21,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=159, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1691649081772_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } 2023-08-10T06:31:21,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1691649081772_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 }, snapshot procedure id = 159 2023-08-10T06:31:21,785 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1691649081772_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:31:21,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=159 2023-08-10T06:31:21,786 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1691649081772_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:31:21,788 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1691649081772_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:31:21,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=159 2023-08-10T06:31:22,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=159 2023-08-10T06:31:22,195 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1691649081772_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:31:22,195 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=160, ppid=159, state=RUNNABLE; SnapshotRegionProcedure 513ad41c456db08f6287ccd38ca6b19c}] 2023-08-10T06:31:22,197 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=160, ppid=159, state=RUNNABLE; SnapshotRegionProcedure 513ad41c456db08f6287ccd38ca6b19c 2023-08-10T06:31:22,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45101 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=160 2023-08-10T06:31:22,348 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1691648868734,,1691648971834.513ad41c456db08f6287ccd38ca6b19c. 2023-08-10T06:31:22,348 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] regionserver.HRegion(2520): Flush status journal for 513ad41c456db08f6287ccd38ca6b19c: 2023-08-10T06:31:22,348 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1691648868734,,1691648971834.513ad41c456db08f6287ccd38ca6b19c. for snapshot_1691649081772_default_test-1691648868734 completed. 2023-08-10T06:31:22,348 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] snapshot.SnapshotManifest(240): Storing 'test-1691648868734,,1691648971834.513ad41c456db08f6287ccd38ca6b19c.' region-info for snapshot=snapshot_1691649081772_default_test-1691648868734 2023-08-10T06:31:22,348 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:31:22,348 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/513ad41c456db08f6287ccd38ca6b19c/f/8eec1fbc8346444d9f3bf9d53b64b2b2_SeqId_4_] hfiles 2023-08-10T06:31:22,348 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/513ad41c456db08f6287ccd38ca6b19c/f/8eec1fbc8346444d9f3bf9d53b64b2b2_SeqId_4_ for snapshot=snapshot_1691649081772_default_test-1691648868734 2023-08-10T06:31:22,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=159 2023-08-10T06:31:22,755 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1691648868734,,1691648971834.513ad41c456db08f6287ccd38ca6b19c. 2023-08-10T06:31:22,755 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=160 2023-08-10T06:31:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster(3994): Remote procedure done, pid=160 2023-08-10T06:31:22,756 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1691649081772_default_test-1691648868734 on region 513ad41c456db08f6287ccd38ca6b19c 2023-08-10T06:31:22,757 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=160, ppid=159, state=RUNNABLE; SnapshotRegionProcedure 513ad41c456db08f6287ccd38ca6b19c 2023-08-10T06:31:22,759 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=160, resume processing ppid=159 2023-08-10T06:31:22,759 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=160, ppid=159, state=SUCCESS; SnapshotRegionProcedure 513ad41c456db08f6287ccd38ca6b19c in 563 msec 2023-08-10T06:31:22,759 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1691649081772_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:31:22,760 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1691649081772_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:31:22,760 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1691649081772_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:31:22,760 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1691649081772_default_test-1691648868734 2023-08-10T06:31:22,761 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_1691649081772_default_test-1691648868734 2023-08-10T06:31:22,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=159 2023-08-10T06:31:23,170 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1691649081772_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:31:23,175 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1691649081772_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:31:23,175 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_1691649081772_default_test-1691648868734 to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691649081772_default_test-1691648868734 2023-08-10T06:31:23,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=159 2023-08-10T06:31:23,991 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1691649081772_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:31:23,991 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1691649081772_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 }, snapshot procedure id = 159 2023-08-10T06:31:23,992 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=159, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1691649081772_default_test-1691648868734 table=test-1691648868734 type=FLUSH ttl=0 } in 2.2080 sec 2023-08-10T06:31:25,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=159 2023-08-10T06:31:25,891 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1691649077443 2023-08-10T06:31:25,891 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-08-10T06:31:25,891 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-08-10T06:31:25,891 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1691649077443_test-1691648868734 2023-08-10T06:31:25,891 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1691649081772_default_test-1691648868734 to hdfs://localhost:45379/backupUT/backup_1691649077443/default/test-1691648868734/ 2023-08-10T06:31:25,891 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-08-10T06:31:25,914 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:45379, inputRoot=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e 2023-08-10T06:31:25,914 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,33705,1691648856595_664871463_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:45379/backupUT/backup_1691649077443/default/test-1691648868734, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:45379/backupUT/backup_1691649077443/default/test-1691648868734/.hbase-snapshot/.tmp/snapshot_1691649081772_default_test-1691648868734 2023-08-10T06:31:25,914 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:45379, inputRoot=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e, snapshotDir=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691649081772_default_test-1691648868734. 2023-08-10T06:31:25,926 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691649081772_default_test-1691648868734 to hdfs://localhost:45379/backupUT/backup_1691649077443/default/test-1691648868734/.hbase-snapshot/.tmp/snapshot_1691649081772_default_test-1691648868734 2023-08-10T06:31:26,343 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:26,343 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:26,344 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:26,344 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:27,324 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/hadoop-7919965148263170082.jar 2023-08-10T06:31:27,325 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:27,325 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:27,326 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:27,326 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:27,327 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:27,327 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:27,327 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:27,328 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:27,328 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:27,329 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:31:27,329 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-08-10T06:31:27,330 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-08-10T06:31:27,330 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-08-10T06:31:27,330 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-08-10T06:31:27,331 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-08-10T06:31:27,331 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-08-10T06:31:27,331 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-08-10T06:31:27,332 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-08-10T06:31:27,332 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-08-10T06:31:27,333 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-08-10T06:31:27,333 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-08-10T06:31:27,333 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:31:27,334 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:31:27,334 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:31:27,335 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:31:27,335 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:31:27,335 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:31:27,336 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:31:27,377 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1691649081772_default_test-1691648868734' hfile list 2023-08-10T06:31:27,379 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-08-10T06:31:27,447 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691649087413/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-08-10T06:31:27,447 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691649087413/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-08-10T06:31:27,497 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-08-10T06:31:27,520 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/513ad41c456db08f6287ccd38ca6b19c/f/8eec1fbc8346444d9f3bf9d53b64b2b2_SeqId_4_ output=hdfs://localhost:45379/backupUT/backup_1691649077443/default/test-1691648868734/archive/data/default/test-1691648868734/513ad41c456db08f6287ccd38ca6b19c/f/8eec1fbc8346444d9f3bf9d53b64b2b2_SeqId_4_ 2023-08-10T06:31:27,520 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8724 (8.5 K) time=0sec 8.320M/sec 2023-08-10T06:31:28,481 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-08-10T06:31:28,482 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-08-10T06:31:28,487 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1691649081772_default_test-1691648868734 2023-08-10T06:31:28,487 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1691649081772_default_test-1691648868734 finished. 2023-08-10T06:31:28,487 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(1014): test-1691648868734 2023-08-10T06:31:28,492 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691649077443 2023-08-10T06:31:28,492 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-08-10T06:31:28,899 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:45379/backupUT/backup_1691649077443/.backup.manifest 2023-08-10T06:31:28,899 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(358): Backup backup_1691649077443 finished: type=FULL,tablelist=test-1691648868734,targetRootDir=hdfs://localhost:45379/backupUT,startts=1691649081588,completets=1691649088492,bytescopied=0 2023-08-10T06:31:28,899 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-08-10T06:31:28,899 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1691649081772_default_test-1691648868734 2023-08-10T06:31:28,899 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1691649081772_default_test-1691648868734" 2023-08-10T06:31:28,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1691649081772_default_test-1691648868734 2023-08-10T06:31:28,902 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1691649081772_default_test-1691648868734 for backup backup_1691649077443 succeeded. 2023-08-10T06:31:28,903 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-08-10T06:31:28,907 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-08-10T06:31:28,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-08-10T06:31:28,909 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-08-10T06:31:28,910 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-08-10T06:31:28,916 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(376): Backup backup_1691649077443 completed. 2023-08-10T06:31:28,916 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:31:28,916 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] backup.TestFullRestore(80): backup complete 2023-08-10T06:31:28,931 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77c81ad1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:31:28,932 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:31:28,934 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:47528, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:31:28,935 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(101): HBase table table1 does not exist. It will be created during restore process 2023-08-10T06:31:28,935 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:45379/backupUT/backup_1691649077443/.backup.manifest 2023-08-10T06:31:28,937 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649077443/.backup.manifest 2023-08-10T06:31:28,938 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:45379/backupUT/backup_1691649077443/.backup.manifest 2023-08-10T06:31:28,940 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649077443/.backup.manifest 2023-08-10T06:31:28,940 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(148): Restoring 'test-1691648868734' to 'table1' from full backup image hdfs://localhost:45379/backupUT/backup_1691649077443/default/test-1691648868734 2023-08-10T06:31:28,947 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] util.RestoreTool(487): Creating target table 'table1' 2023-08-10T06:31:28,947 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] util.RestoreTool(391): Parsing region dir: hdfs://localhost:45379/backupUT/backup_1691649077443/default/test-1691648868734/archive/data/default/test-1691648868734/513ad41c456db08f6287ccd38ca6b19c 2023-08-10T06:31:28,948 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:45379/backupUT/backup_1691649077443/default/test-1691648868734/archive/data/default/test-1691648868734/513ad41c456db08f6287ccd38ca6b19c/f in region [hdfs://localhost:45379/backupUT/backup_1691649077443/default/test-1691648868734/archive/data/default/test-1691648868734/513ad41c456db08f6287ccd38ca6b19c] 2023-08-10T06:31:28,951 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] util.RestoreTool(442): Trying to figure out region boundaries hfile=hdfs://localhost:45379/backupUT/backup_1691649077443/default/test-1691648868734/archive/data/default/test-1691648868734/513ad41c456db08f6287ccd38ca6b19c/f/8eec1fbc8346444d9f3bf9d53b64b2b2_SeqId_4_ first=row0 last=row98 2023-08-10T06:31:28,951 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 8eec1fbc8346444d9f3bf9d53b64b2b2_SeqId_4_ 2023-08-10T06:31:28,952 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:31:28,953 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:50730, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:31:28,954 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:31:28,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=161, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=table1 2023-08-10T06:31:28,957 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=161, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_PRE_OPERATION 2023-08-10T06:31:28,957 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "table1" procId is: 161 2023-08-10T06:31:28,958 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=161, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-08-10T06:31:28,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=161 2023-08-10T06:31:29,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=161 2023-08-10T06:31:29,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=161 2023-08-10T06:31:29,364 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 5ae081d360c450f8a5a67b31647e474f, NAME => 'table1,,1691649088954.5ae081d360c450f8a5a67b31647e474f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e 2023-08-10T06:31:29,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=161 2023-08-10T06:31:29,770 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(888): Instantiated table1,,1691649088954.5ae081d360c450f8a5a67b31647e474f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:31:29,770 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1668): Closing 5ae081d360c450f8a5a67b31647e474f, disabling compactions & flushes 2023-08-10T06:31:29,770 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1690): Closing region table1,,1691649088954.5ae081d360c450f8a5a67b31647e474f. 2023-08-10T06:31:29,770 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on table1,,1691649088954.5ae081d360c450f8a5a67b31647e474f. 2023-08-10T06:31:29,770 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on table1,,1691649088954.5ae081d360c450f8a5a67b31647e474f. after waiting 0 ms 2023-08-10T06:31:29,770 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region table1,,1691649088954.5ae081d360c450f8a5a67b31647e474f. 2023-08-10T06:31:29,770 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1909): Closed table1,,1691649088954.5ae081d360c450f8a5a67b31647e474f. 2023-08-10T06:31:29,770 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1622): Region close journal for 5ae081d360c450f8a5a67b31647e474f: 2023-08-10T06:31:29,771 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=161, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ADD_TO_META 2023-08-10T06:31:29,771 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"table1,,1691649088954.5ae081d360c450f8a5a67b31647e474f.","families":{"info":[{"qualifier":"regioninfo","vlen":40,"tag":[],"timestamp":"1691649089771"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691649089771"}]},"ts":"1691649089771"} 2023-08-10T06:31:29,773 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:31:29,773 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=161, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-08-10T06:31:29,773 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649089773"}]},"ts":"1691649089773"} 2023-08-10T06:31:29,774 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=ENABLING in hbase:meta 2023-08-10T06:31:29,780 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=table1, region=5ae081d360c450f8a5a67b31647e474f, ASSIGN}] 2023-08-10T06:31:29,781 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=table1, region=5ae081d360c450f8a5a67b31647e474f, ASSIGN 2023-08-10T06:31:29,782 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=table1, region=5ae081d360c450f8a5a67b31647e474f, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45101,1691648859755; forceNewPlan=false, retain=false 2023-08-10T06:31:29,936 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=162 updating hbase:meta row=5ae081d360c450f8a5a67b31647e474f, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:31:29,940 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE; OpenRegionProcedure 5ae081d360c450f8a5a67b31647e474f, server=jenkins-hbase3.apache.org,45101,1691648859755}] 2023-08-10T06:31:30,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=161 2023-08-10T06:31:30,095 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] handler.AssignRegionHandler(130): Open table1,,1691649088954.5ae081d360c450f8a5a67b31647e474f. 2023-08-10T06:31:30,095 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(7254): Opening region: {ENCODED => 5ae081d360c450f8a5a67b31647e474f, NAME => 'table1,,1691649088954.5ae081d360c450f8a5a67b31647e474f.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:31:30,095 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:31:30,095 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 5ae081d360c450f8a5a67b31647e474f 2023-08-10T06:31:30,095 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(888): Instantiated table1,,1691649088954.5ae081d360c450f8a5a67b31647e474f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:31:30,095 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(7296): checking encryption for 5ae081d360c450f8a5a67b31647e474f 2023-08-10T06:31:30,095 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(7299): checking classloading for 5ae081d360c450f8a5a67b31647e474f 2023-08-10T06:31:30,096 INFO [StoreOpener-5ae081d360c450f8a5a67b31647e474f-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 5ae081d360c450f8a5a67b31647e474f 2023-08-10T06:31:30,097 INFO [StoreOpener-5ae081d360c450f8a5a67b31647e474f-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5ae081d360c450f8a5a67b31647e474f columnFamilyName f 2023-08-10T06:31:30,098 DEBUG [StoreOpener-5ae081d360c450f8a5a67b31647e474f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:31:30,098 INFO [StoreOpener-5ae081d360c450f8a5a67b31647e474f-1 {}] regionserver.HStore(324): Store=5ae081d360c450f8a5a67b31647e474f/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:31:30,099 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/table1/5ae081d360c450f8a5a67b31647e474f 2023-08-10T06:31:30,099 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/table1/5ae081d360c450f8a5a67b31647e474f 2023-08-10T06:31:30,100 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(1076): writing seq id for 5ae081d360c450f8a5a67b31647e474f 2023-08-10T06:31:30,102 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/table1/5ae081d360c450f8a5a67b31647e474f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:31:30,102 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(1093): Opened 5ae081d360c450f8a5a67b31647e474f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9634208320, jitterRate=-0.10274443030357361}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-08-10T06:31:30,103 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(992): Region open journal for 5ae081d360c450f8a5a67b31647e474f: 2023-08-10T06:31:30,103 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegionServer(2552): Post open deploy tasks for table1,,1691649088954.5ae081d360c450f8a5a67b31647e474f., pid=163, masterSystemTime=1691649090092 2023-08-10T06:31:30,104 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegionServer(2579): Finished post open deploy task for table1,,1691649088954.5ae081d360c450f8a5a67b31647e474f. 2023-08-10T06:31:30,105 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] handler.AssignRegionHandler(158): Opened table1,,1691649088954.5ae081d360c450f8a5a67b31647e474f. 2023-08-10T06:31:30,105 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=162 updating hbase:meta row=5ae081d360c450f8a5a67b31647e474f, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:31:30,107 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=163, resume processing ppid=162 2023-08-10T06:31:30,107 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=163, ppid=162, state=SUCCESS; OpenRegionProcedure 5ae081d360c450f8a5a67b31647e474f, server=jenkins-hbase3.apache.org,45101,1691648859755 in 169 msec 2023-08-10T06:31:30,108 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=162, resume processing ppid=161 2023-08-10T06:31:30,109 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=162, ppid=161, state=SUCCESS; TransitRegionStateProcedure table=table1, region=5ae081d360c450f8a5a67b31647e474f, ASSIGN in 328 msec 2023-08-10T06:31:30,109 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=161, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-08-10T06:31:30,109 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649090109"}]},"ts":"1691649090109"} 2023-08-10T06:31:30,110 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=ENABLED in hbase:meta 2023-08-10T06:31:30,113 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=161, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_POST_OPERATION 2023-08-10T06:31:30,114 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=161, state=SUCCESS; CreateTableProcedure table=table1 in 1.1590 sec 2023-08-10T06:31:30,559 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'table1' 2023-08-10T06:31:31,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=161 2023-08-10T06:31:31,089 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:45379/backupUT/backup_1691649077443/default/test-1691648868734/archive/data/default/test-1691648868734/513ad41c456db08f6287ccd38ca6b19c from hbase tables test-1691648868734 to tables table1 2023-08-10T06:31:31,089 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceRestoreJob(72): Restore test-1691648868734 into table1 2023-08-10T06:31:31,091 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-default-table1-1691649091089 from hdfs://localhost:45379/backupUT/backup_1691649077443/default/test-1691648868734/archive/data/default/test-1691648868734/513ad41c456db08f6287ccd38ca6b19c 2023-08-10T06:31:31,095 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4364d0cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:31:31,096 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:31:31,097 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:48596, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:31:31,098 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-08-10T06:31:31,098 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table table1 2023-08-10T06:31:31,101 DEBUG [hconnection-0x44fa791-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:31:31,102 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:56914, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:31:31,103 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for table1: 2023-08-10T06:31:31,103 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-08-10T06:31:31,103 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/partitions_6a058366-3e6c-4492-bc69-8f9647c10181 2023-08-10T06:31:31,509 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:31,509 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:31,510 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:31,510 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:32,472 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/hadoop-5141158285386937008.jar 2023-08-10T06:31:32,472 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:32,473 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:32,473 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:32,473 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:32,474 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:32,474 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:32,474 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:32,475 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:32,475 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:32,476 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:31:32,476 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-08-10T06:31:32,477 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-08-10T06:31:32,477 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-08-10T06:31:32,477 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-08-10T06:31:32,478 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-08-10T06:31:32,478 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-08-10T06:31:32,479 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-08-10T06:31:32,479 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-08-10T06:31:32,479 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-08-10T06:31:32,480 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-08-10T06:31:32,480 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-08-10T06:31:32,481 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:32,482 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:32,482 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:32,482 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:32,483 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:32,483 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:32,484 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:31:32,484 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: table1 2023-08-10T06:31:32,484 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:31:32,484 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-08-10T06:31:32,484 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-08-10T06:31:32,485 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:31:32,497 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-08-10T06:31:32,570 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691649092537/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-08-10T06:31:32,570 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691649092537/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-08-10T06:31:32,619 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:45379/backupUT/backup_1691649077443/default/test-1691648868734/archive/data/default/test-1691648868734/513ad41c456db08f6287ccd38ca6b19c/f/8eec1fbc8346444d9f3bf9d53b64b2b2_SeqId_4_ 2023-08-10T06:31:32,622 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-08-10T06:31:32,623 DEBUG [LocalJobRunner Map Task Executor #0 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 8eec1fbc8346444d9f3bf9d53b64b2b2_SeqId_4_ 2023-08-10T06:31:32,640 DEBUG [pool-533-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7263bea3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:31:32,641 DEBUG [hconnection-0x7b5da7f0-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:31:32,643 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:56930, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:31:32,646 DEBUG [pool-533-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:31:32,646 DEBUG [pool-533-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-08-10T06:31:32,646 DEBUG [pool-533-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-08-10T06:31:32,650 DEBUG [pool-533-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-default-table1-1691649091089/_temporary/0/_temporary/attempt_local843484064_0025_r_000000_0/f/fa820134f08c4b728d4ccbc8bc76891d is 37, key is row10/f:q1/1691648871386/Put/seqid=0 2023-08-10T06:31:32,651 WARN [Thread-2586 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:45101] Specified favored nodes: [jenkins-hbase3.apache.org:45101] 2023-08-10T06:31:33,619 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-default-table1-1691649091089 2023-08-10T06:31:33,623 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61de2743, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:31:33,624 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:31:33,626 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:56942, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:31:33,631 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-default-table1-1691649091089/_SUCCESS 2023-08-10T06:31:33,634 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:31:33,637 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:48604, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:31:33,644 DEBUG [LoadIncrementalHFiles-0 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: fa820134f08c4b728d4ccbc8bc76891d 2023-08-10T06:31:33,644 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-default-table1-1691649091089/f/fa820134f08c4b728d4ccbc8bc76891d first=Optional[row0] last=Optional[row98] 2023-08-10T06:31:33,645 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=table1,,1691649088954.5ae081d360c450f8a5a67b31647e474f., hostname=jenkins-hbase3.apache.org,45101,1691648859755, seqNum=2 for row with hfile group [{f,hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-default-table1-1691649091089/f/fa820134f08c4b728d4ccbc8bc76891d}] 2023-08-10T06:31:33,650 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-default-table1-1691649091089/f/fa820134f08c4b728d4ccbc8bc76891d for inclusion in 5ae081d360c450f8a5a67b31647e474f/f 2023-08-10T06:31:33,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-08-10T06:31:33,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HStore(625): Region bounds: first= last= 2023-08-10T06:31:33,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: fa820134f08c4b728d4ccbc8bc76891d 2023-08-10T06:31:33,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HRegion(2520): Flush status journal for 5ae081d360c450f8a5a67b31647e474f: 2023-08-10T06:31:33,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/hbase-staging/bulk_output-default-table1-1691649091089/f/fa820134f08c4b728d4ccbc8bc76891d to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/staging/jenkins__table1__i1i5u8bkv3bmss5vm2k848frdelrvrvoq5910jihc00omcilhck8i26v7l1jsak2/f/fa820134f08c4b728d4ccbc8bc76891d 2023-08-10T06:31:33,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/staging/jenkins__table1__i1i5u8bkv3bmss5vm2k848frdelrvrvoq5910jihc00omcilhck8i26v7l1jsak2/f/fa820134f08c4b728d4ccbc8bc76891d as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/table1/5ae081d360c450f8a5a67b31647e474f/f/53f26dd3743049139bb69e698db4a0d5_SeqId_4_ 2023-08-10T06:31:33,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@642d6db7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:31:33,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:31:33,661 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:48616, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:31:33,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:31:33,665 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:56954, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:31:33,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:31:33,676 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/staging/jenkins__table1__i1i5u8bkv3bmss5vm2k848frdelrvrvoq5910jihc00omcilhck8i26v7l1jsak2/f/fa820134f08c4b728d4ccbc8bc76891d into 5ae081d360c450f8a5a67b31647e474f/f as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/table1/5ae081d360c450f8a5a67b31647e474f/f/53f26dd3743049139bb69e698db4a0d5_SeqId_4_ - updating store file list. 2023-08-10T06:31:33,681 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/table1/5ae081d360c450f8a5a67b31647e474f/f/53f26dd3743049139bb69e698db4a0d5_SeqId_4_ into 5ae081d360c450f8a5a67b31647e474f/f 2023-08-10T06:31:33,681 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/staging/jenkins__table1__i1i5u8bkv3bmss5vm2k848frdelrvrvoq5910jihc00omcilhck8i26v7l1jsak2/f/fa820134f08c4b728d4ccbc8bc76891d into 5ae081d360c450f8a5a67b31647e474f/f (new location: hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/table1/5ae081d360c450f8a5a67b31647e474f/f/53f26dd3743049139bb69e698db4a0d5_SeqId_4_) 2023-08-10T06:31:33,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/staging/jenkins__table1__i1i5u8bkv3bmss5vm2k848frdelrvrvoq5910jihc00omcilhck8i26v7l1jsak2/f/fa820134f08c4b728d4ccbc8bc76891d 2023-08-10T06:31:33,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a0da528, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:31:33,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:31:33,689 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:48628, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-08-10T06:31:33,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:31:33,693 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:56960, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-08-10T06:31:33,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45101 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:31:33,709 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:31:33,710 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-08-10T06:31:33,710 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-08-10T06:31:33,710 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(233): Backup: backup_1691649077443 hdfs://localhost:45379/backupUT/backup_1691649077443/default/test-1691648868734/ 2023-08-10T06:31:33,710 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-08-10T06:31:33,710 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:31:33,711 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster$12(2663): Client=jenkins//172.31.12.81 disable table1 2023-08-10T06:31:33,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=164, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=table1 2023-08-10T06:31:33,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=164 2023-08-10T06:31:33,715 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649093714"}]},"ts":"1691649093714"} 2023-08-10T06:31:33,715 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=DISABLING in hbase:meta 2023-08-10T06:31:33,717 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(293): Set table1 to state=DISABLING 2023-08-10T06:31:33,718 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=table1, region=5ae081d360c450f8a5a67b31647e474f, UNASSIGN}] 2023-08-10T06:31:33,719 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=165, ppid=164, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=table1, region=5ae081d360c450f8a5a67b31647e474f, UNASSIGN 2023-08-10T06:31:33,720 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=165 updating hbase:meta row=5ae081d360c450f8a5a67b31647e474f, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:31:33,721 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=166, ppid=165, state=RUNNABLE; CloseRegionProcedure 5ae081d360c450f8a5a67b31647e474f, server=jenkins-hbase3.apache.org,45101,1691648859755}] 2023-08-10T06:31:33,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=164 2023-08-10T06:31:33,872 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] handler.UnassignRegionHandler(119): Close 5ae081d360c450f8a5a67b31647e474f 2023-08-10T06:31:33,872 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegion(1668): Closing 5ae081d360c450f8a5a67b31647e474f, disabling compactions & flushes 2023-08-10T06:31:33,872 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegion(1690): Closing region table1,,1691649088954.5ae081d360c450f8a5a67b31647e474f. 2023-08-10T06:31:33,872 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegion(1711): Waiting without time limit for close lock on table1,,1691649088954.5ae081d360c450f8a5a67b31647e474f. 2023-08-10T06:31:33,872 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegion(1778): Acquired close lock on table1,,1691649088954.5ae081d360c450f8a5a67b31647e474f. after waiting 0 ms 2023-08-10T06:31:33,872 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegion(1788): Updates disabled for region table1,,1691649088954.5ae081d360c450f8a5a67b31647e474f. 2023-08-10T06:31:33,874 DEBUG [StoreFileCloser-5ae081d360c450f8a5a67b31647e474f-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 53f26dd3743049139bb69e698db4a0d5_SeqId_4_ 2023-08-10T06:31:33,879 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/table1/5ae081d360c450f8a5a67b31647e474f/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2023-08-10T06:31:33,879 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:31:33,879 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegion(1909): Closed table1,,1691649088954.5ae081d360c450f8a5a67b31647e474f. 2023-08-10T06:31:33,879 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegion(1622): Region close journal for 5ae081d360c450f8a5a67b31647e474f: 2023-08-10T06:31:33,881 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] handler.UnassignRegionHandler(163): Closed 5ae081d360c450f8a5a67b31647e474f 2023-08-10T06:31:33,882 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=165 updating hbase:meta row=5ae081d360c450f8a5a67b31647e474f, regionState=CLOSED 2023-08-10T06:31:33,884 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=166, resume processing ppid=165 2023-08-10T06:31:33,884 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=166, ppid=165, state=SUCCESS; CloseRegionProcedure 5ae081d360c450f8a5a67b31647e474f, server=jenkins-hbase3.apache.org,45101,1691648859755 in 162 msec 2023-08-10T06:31:33,885 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=165, resume processing ppid=164 2023-08-10T06:31:33,885 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=165, ppid=164, state=SUCCESS; TransitRegionStateProcedure table=table1, region=5ae081d360c450f8a5a67b31647e474f, UNASSIGN in 166 msec 2023-08-10T06:31:33,885 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649093885"}]},"ts":"1691649093885"} 2023-08-10T06:31:33,886 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=DISABLED in hbase:meta 2023-08-10T06:31:33,888 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(305): Set table1 to state=DISABLED 2023-08-10T06:31:33,889 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=164, state=SUCCESS; DisableTableProcedure table=table1 in 177 msec 2023-08-10T06:31:34,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=164 2023-08-10T06:31:34,017 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster$5(2447): Client=jenkins//172.31.12.81 delete table1 2023-08-10T06:31:34,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=167, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=table1 2023-08-10T06:31:34,019 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=167, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=table1 2023-08-10T06:31:34,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=167 2023-08-10T06:31:34,020 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=167, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=table1 2023-08-10T06:31:34,021 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/table1/5ae081d360c450f8a5a67b31647e474f 2023-08-10T06:31:34,023 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/table1/5ae081d360c450f8a5a67b31647e474f/f, FileablePath, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/table1/5ae081d360c450f8a5a67b31647e474f/recovered.edits] 2023-08-10T06:31:34,027 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/table1/5ae081d360c450f8a5a67b31647e474f/f/53f26dd3743049139bb69e698db4a0d5_SeqId_4_ to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/archive/data/default/table1/5ae081d360c450f8a5a67b31647e474f/f/53f26dd3743049139bb69e698db4a0d5_SeqId_4_ 2023-08-10T06:31:34,030 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/table1/5ae081d360c450f8a5a67b31647e474f/recovered.edits/6.seqid to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/archive/data/default/table1/5ae081d360c450f8a5a67b31647e474f/recovered.edits/6.seqid 2023-08-10T06:31:34,031 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/table1/5ae081d360c450f8a5a67b31647e474f 2023-08-10T06:31:34,031 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(293): Archived table1 regions 2023-08-10T06:31:34,033 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=167, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=table1 2023-08-10T06:31:34,035 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of table1 from hbase:meta 2023-08-10T06:31:34,037 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(387): Removing 'table1' descriptor. 2023-08-10T06:31:34,038 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=167, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=table1 2023-08-10T06:31:34,038 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(377): Removing 'table1' from region states. 2023-08-10T06:31:34,038 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"table1,,1691649088954.5ae081d360c450f8a5a67b31647e474f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1691649094038"}]},"ts":"9223372036854775807"} 2023-08-10T06:31:34,039 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-08-10T06:31:34,039 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 5ae081d360c450f8a5a67b31647e474f, NAME => 'table1,,1691649088954.5ae081d360c450f8a5a67b31647e474f.', STARTKEY => '', ENDKEY => ''}] 2023-08-10T06:31:34,039 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(381): Marking 'table1' as deleted. 2023-08-10T06:31:34,039 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1691649094039"}]},"ts":"9223372036854775807"} 2023-08-10T06:31:34,041 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1678): Deleted table table1 state from META 2023-08-10T06:31:34,043 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(127): Finished pid=167, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=table1 2023-08-10T06:31:34,043 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=167, state=SUCCESS; DeleteTableProcedure table=table1 in 26 msec 2023-08-10T06:31:34,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=167 2023-08-10T06:31:34,136 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreSingleCommand Thread=604 (was 602) - Thread LEAK? -, OpenFileDescriptor=684 (was 686), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=98 (was 106), ProcessCount=170 (was 171), AvailableMemoryMB=1569 (was 1619) 2023-08-10T06:31:34,137 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(130): Thread=604 is superior to 500 2023-08-10T06:31:34,151 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreMultipleDNE Thread=604, OpenFileDescriptor=684, MaxFileDescriptor=60000, SystemLoadAverage=98, ProcessCount=170, AvailableMemoryMB=1568 2023-08-10T06:31:34,151 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(130): Thread=604 is superior to 500 2023-08-10T06:31:34,151 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] backup.TestFullRestore(302): test restore fails on multiple tables that do not exist 2023-08-10T06:31:34,157 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@284c3576, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:31:34,160 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:31:34,162 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:56964, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:31:34,163 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:31:34,165 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:48634, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:31:34,173 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-08-10T06:31:34,177 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:31:34,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691649094177 (current time:1691649094177). 2023-08-10T06:31:34,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:31:34,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-08-10T06:31:34,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@745fea76, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:31:34,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:31:34,184 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:56968, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:31:34,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:31:34,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:31:34,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=168, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:31:34,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 168 2023-08-10T06:31:34,187 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:31:34,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=168 2023-08-10T06:31:34,187 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:31:34,189 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:31:34,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=168 2023-08-10T06:31:34,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=168 2023-08-10T06:31:34,594 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:31:34,594 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72}] 2023-08-10T06:31:34,596 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=169, ppid=168, state=RUNNABLE; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:31:34,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45101 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=169 2023-08-10T06:31:34,748 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. 2023-08-10T06:31:34,748 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.HRegion(2819): Flushing b9c493f2c9fbef45de19e8268a534e72 2/2 column families, dataSize=1.14 KB heapSize=2.22 KB 2023-08-10T06:31:34,751 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/meta/07b4147d42b3429baac41e09e439933a is 169, key is trslm:hdfs://localhost:45379/backupUT\x00test-1691648868734/meta:log-roll-map/1691649088489/Put/seqid=0 2023-08-10T06:31:34,755 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=451 B at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/meta/07b4147d42b3429baac41e09e439933a 2023-08-10T06:31:34,760 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 07b4147d42b3429baac41e09e439933a 2023-08-10T06:31:34,763 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/session/ea1ed59991114105a8ce717031f62056 is 452, key is session:backup_1691649077443/session:context/1691649088910/Put/seqid=0 2023-08-10T06:31:34,768 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=721 B at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/session/ea1ed59991114105a8ce717031f62056 2023-08-10T06:31:34,773 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: ea1ed59991114105a8ce717031f62056 2023-08-10T06:31:34,774 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/meta/07b4147d42b3429baac41e09e439933a as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/07b4147d42b3429baac41e09e439933a 2023-08-10T06:31:34,782 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/07b4147d42b3429baac41e09e439933a, entries=4, sequenceid=138, filesize=5.5 K 2023-08-10T06:31:34,783 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/session/ea1ed59991114105a8ce717031f62056 as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/ea1ed59991114105a8ce717031f62056 2023-08-10T06:31:34,789 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/ea1ed59991114105a8ce717031f62056, entries=2, sequenceid=138, filesize=5.4 K 2023-08-10T06:31:34,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=168 2023-08-10T06:31:34,789 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1172, heapSize ~2.19 KB/2240, currentSize=0 B/0 for b9c493f2c9fbef45de19e8268a534e72 in 41ms, sequenceid=138, compaction requested=true 2023-08-10T06:31:34,790 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.HRegion(2520): Flush status journal for b9c493f2c9fbef45de19e8268a534e72: 2023-08-10T06:31:34,790 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. for snapshot_backup_system completed. 2023-08-10T06:31:34,790 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72.' region-info for snapshot=snapshot_backup_system 2023-08-10T06:31:34,790 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:31:34,790 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/bc10682e7f36491d941e2fe62e2e4100, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/9d9a4feb1c4e44d99cc63e3f68878731, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/f04a99990b2c429eaf1096ef413be3b8, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/9e97dc38b66b476f952990b23fa0a3ca, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/930c578fe3f543e1b6308abfb31c7246, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/24713cefebe0499c812c6d209dea0fd6, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/0ae449898702485faa63081b0f3f028a, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/cec52c43e8824ebfbbc2cf17b7a9ec5f, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/8f5b1bc0de87458e96bd38212599dcbd, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/184a27fc671b4e9085d56df41f1234f5, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/225d36cdae6a48eb86402cbd55c85ab2, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/07b4147d42b3429baac41e09e439933a] hfiles 2023-08-10T06:31:34,790 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (1/12): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/bc10682e7f36491d941e2fe62e2e4100 for snapshot=snapshot_backup_system 2023-08-10T06:31:34,790 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (2/12): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/9d9a4feb1c4e44d99cc63e3f68878731 for snapshot=snapshot_backup_system 2023-08-10T06:31:34,790 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (3/12): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/f04a99990b2c429eaf1096ef413be3b8 for snapshot=snapshot_backup_system 2023-08-10T06:31:34,790 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (4/12): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/9e97dc38b66b476f952990b23fa0a3ca for snapshot=snapshot_backup_system 2023-08-10T06:31:34,790 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (5/12): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/930c578fe3f543e1b6308abfb31c7246 for snapshot=snapshot_backup_system 2023-08-10T06:31:34,790 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (6/12): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/24713cefebe0499c812c6d209dea0fd6 for snapshot=snapshot_backup_system 2023-08-10T06:31:34,790 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (7/12): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/0ae449898702485faa63081b0f3f028a for snapshot=snapshot_backup_system 2023-08-10T06:31:34,790 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (8/12): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/cec52c43e8824ebfbbc2cf17b7a9ec5f for snapshot=snapshot_backup_system 2023-08-10T06:31:34,790 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (9/12): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/8f5b1bc0de87458e96bd38212599dcbd for snapshot=snapshot_backup_system 2023-08-10T06:31:34,790 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (10/12): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/184a27fc671b4e9085d56df41f1234f5 for snapshot=snapshot_backup_system 2023-08-10T06:31:34,790 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (11/12): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/225d36cdae6a48eb86402cbd55c85ab2 for snapshot=snapshot_backup_system 2023-08-10T06:31:34,790 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (12/12): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/07b4147d42b3429baac41e09e439933a for snapshot=snapshot_backup_system 2023-08-10T06:31:34,790 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/e88678d5b4b14eecabac817daf12539e, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/37561b9a8ae2406bb6d053c33aeff31c, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/883911c19cab473f8bf801f934c90bc1, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/0c7705a45c554d6a97fa37fac319ceff, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/398a15bfffb84acfadb1be252c2e133a, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/adceb6320b02451abd02beed6c69c000, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/68fd0cf6c3094cc7af1a5176829fd9a4, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/40f9fbdeac5b49dbb654e429c177708d, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/75226fb0202e42819224b5e5920a02f2, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/3702fe0322c94bcc85372b5eb9c99029, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/8a873f56ce4e4af992eb65b5390f76c2, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/1dd3913d29eb48739f0c6f2120db1ded, hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/ea1ed59991114105a8ce717031f62056] hfiles 2023-08-10T06:31:34,790 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (1/13): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/e88678d5b4b14eecabac817daf12539e for snapshot=snapshot_backup_system 2023-08-10T06:31:34,790 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (2/13): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/37561b9a8ae2406bb6d053c33aeff31c for snapshot=snapshot_backup_system 2023-08-10T06:31:34,790 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (3/13): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/883911c19cab473f8bf801f934c90bc1 for snapshot=snapshot_backup_system 2023-08-10T06:31:34,790 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (4/13): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/0c7705a45c554d6a97fa37fac319ceff for snapshot=snapshot_backup_system 2023-08-10T06:31:34,790 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (5/13): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/398a15bfffb84acfadb1be252c2e133a for snapshot=snapshot_backup_system 2023-08-10T06:31:34,790 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (6/13): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/adceb6320b02451abd02beed6c69c000 for snapshot=snapshot_backup_system 2023-08-10T06:31:34,790 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (7/13): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/68fd0cf6c3094cc7af1a5176829fd9a4 for snapshot=snapshot_backup_system 2023-08-10T06:31:34,790 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (8/13): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/40f9fbdeac5b49dbb654e429c177708d for snapshot=snapshot_backup_system 2023-08-10T06:31:34,790 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (9/13): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/75226fb0202e42819224b5e5920a02f2 for snapshot=snapshot_backup_system 2023-08-10T06:31:34,790 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (10/13): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/3702fe0322c94bcc85372b5eb9c99029 for snapshot=snapshot_backup_system 2023-08-10T06:31:34,790 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (11/13): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/8a873f56ce4e4af992eb65b5390f76c2 for snapshot=snapshot_backup_system 2023-08-10T06:31:34,790 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (12/13): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/1dd3913d29eb48739f0c6f2120db1ded for snapshot=snapshot_backup_system 2023-08-10T06:31:34,791 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (13/13): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/ea1ed59991114105a8ce717031f62056 for snapshot=snapshot_backup_system 2023-08-10T06:31:34,795 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. 2023-08-10T06:31:34,795 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=169 2023-08-10T06:31:34,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster(3994): Remote procedure done, pid=169 2023-08-10T06:31:34,795 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:31:34,797 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=169, ppid=168, state=RUNNABLE; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:31:34,798 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=169, resume processing ppid=168 2023-08-10T06:31:34,798 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=169, ppid=168, state=SUCCESS; SnapshotRegionProcedure b9c493f2c9fbef45de19e8268a534e72 in 203 msec 2023-08-10T06:31:34,798 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:31:34,799 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:31:34,799 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:31:34,799 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-08-10T06:31:34,800 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_backup_system 2023-08-10T06:31:35,210 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:31:35,225 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:31:35,226 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_backup_system 2023-08-10T06:31:35,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=168 2023-08-10T06:31:35,633 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-08-10T06:31:36,042 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:31:36,042 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 168 2023-08-10T06:31:36,043 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=168, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.8570 sec 2023-08-10T06:31:36,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=168 2023-08-10T06:31:36,291 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(115): Backup backup_1691649094157 started at 1691649096291. 2023-08-10T06:31:36,292 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(119): Backup session backup_1691649094157 has been started. 2023-08-10T06:31:36,297 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-08-10T06:31:36,297 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-08-10T06:31:36,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-08-10T06:31:36,297 INFO [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-08-10T06:31:36,297 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-08-10T06:31:36,299 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-08-10T06:31:36,299 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-08-10T06:31:36,299 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:31:36,301 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:31:36,301 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:31:36,301 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:31:36,301 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-08-10T06:31:36,301 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-08-10T06:31:36,302 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-08-10T06:31:36,302 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:31:36,302 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-08-10T06:31:36,302 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-08-10T06:31:36,302 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-08-10T06:31:36,302 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-08-10T06:31:36,302 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-08-10T06:31:36,303 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-08-10T06:31:36,303 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-08-10T06:31:36,303 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,45101,1691648859755' joining acquired barrier for procedure (rolllog) in zk 2023-08-10T06:31:36,305 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-08-10T06:31:36,306 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:31:36,306 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:31:36,306 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:31:36,306 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-08-10T06:31:36,306 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:31:36,306 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:31:36,306 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:31:36,307 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:31:36,307 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:31:36,307 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,45101,1691648859755' joining acquired barrier for procedure 'rolllog' on coordinator 2023-08-10T06:31:36,307 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@7aac30c9[Count = 0] remaining members to acquire global barrier 2023-08-10T06:31:36,307 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-08-10T06:31:36,307 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-08-10T06:31:36,308 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-08-10T06:31:36,308 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-08-10T06:31:36,308 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-08-10T06:31:36,309 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-08-10T06:31:36,309 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-08-10T06:31:36,309 DEBUG [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:31:36,309 INFO [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1691649081622 highest: 1691649081622 on jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:31:36,309 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta:.meta(num 1691649081605) roll requested 2023-08-10T06:31:36,320 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691649096309.meta, exclude list is [], retry=0 2023-08-10T06:31:36,323 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK] 2023-08-10T06:31:36,324 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691649081605.meta with entries=12, filesize=2.74 KB; new WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691649096309.meta 2023-08-10T06:31:36,324 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK]] 2023-08-10T06:31:36,324 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta.1691649081605.meta is not closed yet, will try archiving it next time 2023-08-10T06:31:36,324 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C45101%2C1691648859755:(num 1691649081622) roll requested 2023-08-10T06:31:36,336 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691649096325, exclude list is [], retry=0 2023-08-10T06:31:36,338 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK] 2023-08-10T06:31:36,339 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691649081622 with entries=13, filesize=3.13 KB; new WAL /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691649096325 2023-08-10T06:31:36,339 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43219,DS-d83403a9-290e-4799-b60c-fa534525be24,DISK]] 2023-08-10T06:31:36,339 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691649081622 is not closed yet, will try archiving it next time 2023-08-10T06:31:36,340 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/WALs/jenkins-hbase3.apache.org,45101,1691648859755/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691649070128 to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/oldWALs/jenkins-hbase3.apache.org%2C45101%2C1691648859755.1691649070128 2023-08-10T06:31:36,349 DEBUG [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-08-10T06:31:36,349 INFO [rs(jenkins-hbase3.apache.org,45101,1691648859755)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1691649096325 on jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:31:36,355 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-08-10T06:31:36,355 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,45101,1691648859755' in zk 2023-08-10T06:31:36,357 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-08-10T06:31:36,357 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:31:36,357 DEBUG [member: 'jenkins-hbase3.apache.org,45101,1691648859755' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-08-10T06:31:36,357 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:31:36,358 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:31:36,358 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:31:36,358 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:31:36,359 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:31:36,359 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:31:36,359 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:31:36,359 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:31:36,360 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:31:36,360 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:31:36,360 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,45101,1691648859755': 2023-08-10T06:31:36,360 INFO [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-08-10T06:31:36,360 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,45101,1691648859755' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-08-10T06:31:36,361 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-08-10T06:31:36,361 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-08-10T06:31:36,361 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-08-10T06:31:36,361 INFO [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-08-10T06:31:36,363 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-08-10T06:31:36,363 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-08-10T06:31:36,363 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:31:36,363 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:31:36,363 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:31:36,363 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:31:36,363 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:31:36,363 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:31:36,364 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:31:36,364 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:31:36,364 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:31:36,364 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:31:36,364 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:31:36,365 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:31:36,365 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:31:36,365 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:31:36,369 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:31:36,369 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:31:36,369 DEBUG [(jenkins-hbase3.apache.org,33705,1691648856595)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:31:36,369 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-08-10T06:31:36,369 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-08-10T06:31:36,370 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:31:36,370 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:31:36,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-08-10T06:31:36,473 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1691649096473_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } 2023-08-10T06:31:36,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691649096473 (current time:1691649096473). 2023-08-10T06:31:36,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:31:36,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1691649096473_ns2_test-16916488687341 VERSION not specified, setting to 2 2023-08-10T06:31:36,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41dcbfd2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:31:36,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:31:36,482 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:56980, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:31:36,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:31:36,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:31:36,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=170, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1691649096473_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } 2023-08-10T06:31:36,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1691649096473_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 }, snapshot procedure id = 170 2023-08-10T06:31:36,486 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1691649096473_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:31:36,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=170 2023-08-10T06:31:36,486 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1691649096473_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:31:36,488 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1691649096473_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:31:36,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=170 2023-08-10T06:31:36,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=170 2023-08-10T06:31:36,894 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1691649096473_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:31:36,895 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE; SnapshotRegionProcedure 25a1c5b32e608be67dca00df059e0053}] 2023-08-10T06:31:36,896 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=171, ppid=170, state=RUNNABLE; SnapshotRegionProcedure 25a1c5b32e608be67dca00df059e0053 2023-08-10T06:31:37,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45101 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=171 2023-08-10T06:31:37,048 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-16916488687341,,1691649034386.25a1c5b32e608be67dca00df059e0053. 2023-08-10T06:31:37,048 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] regionserver.HRegion(2520): Flush status journal for 25a1c5b32e608be67dca00df059e0053: 2023-08-10T06:31:37,048 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-16916488687341,,1691649034386.25a1c5b32e608be67dca00df059e0053. for snapshot_1691649096473_ns2_test-16916488687341 completed. 2023-08-10T06:31:37,048 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] snapshot.SnapshotManifest(240): Storing 'ns2:test-16916488687341,,1691649034386.25a1c5b32e608be67dca00df059e0053.' region-info for snapshot=snapshot_1691649096473_ns2_test-16916488687341 2023-08-10T06:31:37,048 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:31:37,048 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/25a1c5b32e608be67dca00df059e0053/f/06cc12c6a1d04d99aa771c07eb41201c_SeqId_4_] hfiles 2023-08-10T06:31:37,048 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/25a1c5b32e608be67dca00df059e0053/f/06cc12c6a1d04d99aa771c07eb41201c_SeqId_4_ for snapshot=snapshot_1691649096473_ns2_test-16916488687341 2023-08-10T06:31:37,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=170 2023-08-10T06:31:37,453 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-16916488687341,,1691649034386.25a1c5b32e608be67dca00df059e0053. 2023-08-10T06:31:37,453 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=171 2023-08-10T06:31:37,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster(3994): Remote procedure done, pid=171 2023-08-10T06:31:37,453 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1691649096473_ns2_test-16916488687341 on region 25a1c5b32e608be67dca00df059e0053 2023-08-10T06:31:37,455 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=171, ppid=170, state=RUNNABLE; SnapshotRegionProcedure 25a1c5b32e608be67dca00df059e0053 2023-08-10T06:31:37,456 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=171, resume processing ppid=170 2023-08-10T06:31:37,456 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1691649096473_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:31:37,456 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=171, ppid=170, state=SUCCESS; SnapshotRegionProcedure 25a1c5b32e608be67dca00df059e0053 in 560 msec 2023-08-10T06:31:37,457 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1691649096473_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:31:37,457 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1691649096473_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:31:37,457 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1691649096473_ns2_test-16916488687341 2023-08-10T06:31:37,458 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_1691649096473_ns2_test-16916488687341 2023-08-10T06:31:37,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=170 2023-08-10T06:31:37,867 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1691649096473_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:31:37,872 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1691649096473_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:31:37,872 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_1691649096473_ns2_test-16916488687341 to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691649096473_ns2_test-16916488687341 2023-08-10T06:31:38,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=170 2023-08-10T06:31:38,686 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1691649096473_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:31:38,686 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1691649096473_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 }, snapshot procedure id = 170 2023-08-10T06:31:38,687 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=170, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1691649096473_ns2_test-16916488687341 table=ns2:test-16916488687341 type=FLUSH ttl=0 } in 2.2020 sec 2023-08-10T06:31:40,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=170 2023-08-10T06:31:40,592 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1691649100592_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } 2023-08-10T06:31:40,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691649100592 (current time:1691649100592). 2023-08-10T06:31:40,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:31:40,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1691649100592_ns3_test-16916488687342 VERSION not specified, setting to 2 2023-08-10T06:31:40,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e93cb80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:31:40,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:31:40,601 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:44404, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:31:40,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:31:40,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:31:40,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure2.ProcedureExecutor(1033): Stored pid=172, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1691649100592_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } 2023-08-10T06:31:40,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1691649100592_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 }, snapshot procedure id = 172 2023-08-10T06:31:40,604 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1691649100592_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:31:40,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=172 2023-08-10T06:31:40,605 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1691649100592_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:31:40,606 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1691649100592_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:31:40,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=172 2023-08-10T06:31:40,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=172 2023-08-10T06:31:41,011 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1691649100592_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:31:41,011 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE; SnapshotRegionProcedure 547cc5444af6a8eb48a29d4d03cf49dd}] 2023-08-10T06:31:41,013 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=173, ppid=172, state=RUNNABLE; SnapshotRegionProcedure 547cc5444af6a8eb48a29d4d03cf49dd 2023-08-10T06:31:41,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45101 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=173 2023-08-10T06:31:41,164 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=173}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-16916488687342,,1691649039444.547cc5444af6a8eb48a29d4d03cf49dd. 2023-08-10T06:31:41,164 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=173}] regionserver.HRegion(2520): Flush status journal for 547cc5444af6a8eb48a29d4d03cf49dd: 2023-08-10T06:31:41,164 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=173}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-16916488687342,,1691649039444.547cc5444af6a8eb48a29d4d03cf49dd. for snapshot_1691649100592_ns3_test-16916488687342 completed. 2023-08-10T06:31:41,165 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=173}] snapshot.SnapshotManifest(240): Storing 'ns3:test-16916488687342,,1691649039444.547cc5444af6a8eb48a29d4d03cf49dd.' region-info for snapshot=snapshot_1691649100592_ns3_test-16916488687342 2023-08-10T06:31:41,165 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=173}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:31:41,165 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=173}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-08-10T06:31:41,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=172 2023-08-10T06:31:41,570 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=173}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-16916488687342,,1691649039444.547cc5444af6a8eb48a29d4d03cf49dd. 2023-08-10T06:31:41,570 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=173 2023-08-10T06:31:41,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.HMaster(3994): Remote procedure done, pid=173 2023-08-10T06:31:41,570 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1691649100592_ns3_test-16916488687342 on region 547cc5444af6a8eb48a29d4d03cf49dd 2023-08-10T06:31:41,572 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=173, ppid=172, state=RUNNABLE; SnapshotRegionProcedure 547cc5444af6a8eb48a29d4d03cf49dd 2023-08-10T06:31:41,573 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=173, resume processing ppid=172 2023-08-10T06:31:41,574 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1691649100592_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:31:41,574 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=173, ppid=172, state=SUCCESS; SnapshotRegionProcedure 547cc5444af6a8eb48a29d4d03cf49dd in 561 msec 2023-08-10T06:31:41,574 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1691649100592_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:31:41,575 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1691649100592_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:31:41,575 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1691649100592_ns3_test-16916488687342 2023-08-10T06:31:41,575 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_1691649100592_ns3_test-16916488687342 2023-08-10T06:31:41,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=172 2023-08-10T06:31:41,983 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1691649100592_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:31:41,987 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1691649100592_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:31:41,987 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/.tmp/snapshot_1691649100592_ns3_test-16916488687342 to hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691649100592_ns3_test-16916488687342 2023-08-10T06:31:42,017 DEBUG [regionserver/jenkins-hbase3:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.7692307692307693, tune throughput to 88.46 MB/second 2023-08-10T06:31:42,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=172 2023-08-10T06:31:42,804 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1691649100592_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:31:42,804 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1691649100592_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 }, snapshot procedure id = 172 2023-08-10T06:31:42,805 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=172, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1691649100592_ns3_test-16916488687342 table=ns3:test-16916488687342 type=FLUSH ttl=0 } in 2.2020 sec 2023-08-10T06:31:44,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=172 2023-08-10T06:31:44,710 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1691649094157 2023-08-10T06:31:44,710 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-08-10T06:31:44,710 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2023-08-10T06:31:44,710 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1691649094157_ns3:test-16916488687342 2023-08-10T06:31:44,710 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1691649100592_ns3_test-16916488687342 to hdfs://localhost:45379/backupUT/backup_1691649094157/ns3/test-16916488687342/ 2023-08-10T06:31:44,710 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-08-10T06:31:44,732 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:45379, inputRoot=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e 2023-08-10T06:31:44,732 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,33705,1691648856595_981828138_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:45379/backupUT/backup_1691649094157/ns3/test-16916488687342, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:45379/backupUT/backup_1691649094157/ns3/test-16916488687342/.hbase-snapshot/.tmp/snapshot_1691649100592_ns3_test-16916488687342 2023-08-10T06:31:44,732 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:45379, inputRoot=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e, snapshotDir=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691649100592_ns3_test-16916488687342. 2023-08-10T06:31:44,740 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691649100592_ns3_test-16916488687342 to hdfs://localhost:45379/backupUT/backup_1691649094157/ns3/test-16916488687342/.hbase-snapshot/.tmp/snapshot_1691649100592_ns3_test-16916488687342 2023-08-10T06:31:44,764 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:44,765 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:44,766 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:44,766 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:45,731 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/hadoop-8606831485763128664.jar 2023-08-10T06:31:45,731 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:45,731 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:45,732 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:45,732 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:45,733 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:45,733 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:45,733 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:45,733 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:45,734 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:45,734 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:31:45,735 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-08-10T06:31:45,735 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-08-10T06:31:45,736 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-08-10T06:31:45,736 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-08-10T06:31:45,736 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-08-10T06:31:45,737 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-08-10T06:31:45,737 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-08-10T06:31:45,737 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-08-10T06:31:45,738 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-08-10T06:31:45,738 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-08-10T06:31:45,738 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-08-10T06:31:45,739 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:31:45,739 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:31:45,740 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:31:45,740 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:31:45,740 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:31:45,741 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:31:45,741 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:31:45,784 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1691649100592_ns3_test-16916488687342' hfile list 2023-08-10T06:31:45,854 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691649105819/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-08-10T06:31:45,854 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691649105819/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-08-10T06:31:46,882 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-08-10T06:31:46,883 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-08-10T06:31:46,888 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1691649100592_ns3_test-16916488687342 2023-08-10T06:31:46,888 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1691649100592_ns3_test-16916488687342 finished. 2023-08-10T06:31:46,888 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1691649094157_ns2:test-16916488687341 2023-08-10T06:31:46,888 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1691649096473_ns2_test-16916488687341 to hdfs://localhost:45379/backupUT/backup_1691649094157/ns2/test-16916488687341/ 2023-08-10T06:31:46,888 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-08-10T06:31:46,911 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:45379, inputRoot=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e 2023-08-10T06:31:46,911 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,33705,1691648856595_-1800439532_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:45379/backupUT/backup_1691649094157/ns2/test-16916488687341, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:45379/backupUT/backup_1691649094157/ns2/test-16916488687341/.hbase-snapshot/.tmp/snapshot_1691649096473_ns2_test-16916488687341 2023-08-10T06:31:46,911 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:45379, inputRoot=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e, snapshotDir=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691649096473_ns2_test-16916488687341. 2023-08-10T06:31:46,916 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/.hbase-snapshot/snapshot_1691649096473_ns2_test-16916488687341 to hdfs://localhost:45379/backupUT/backup_1691649094157/ns2/test-16916488687341/.hbase-snapshot/.tmp/snapshot_1691649096473_ns2_test-16916488687341 2023-08-10T06:31:46,933 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:46,934 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:46,934 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:46,934 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:47,892 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/hadoop-8585256368079553789.jar 2023-08-10T06:31:47,892 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:47,892 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:47,893 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:47,893 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:47,894 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:47,894 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:47,894 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:47,894 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:47,895 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:47,895 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:31:47,896 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-08-10T06:31:47,896 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-08-10T06:31:47,897 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-08-10T06:31:47,897 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-08-10T06:31:47,897 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-08-10T06:31:47,898 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-08-10T06:31:47,898 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-08-10T06:31:47,898 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-08-10T06:31:47,899 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-08-10T06:31:47,899 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-08-10T06:31:47,899 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-08-10T06:31:47,900 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:31:47,900 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:31:47,901 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:31:47,901 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:31:47,901 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:31:47,902 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:31:47,902 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:31:47,945 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1691649096473_ns2_test-16916488687341' hfile list 2023-08-10T06:31:47,947 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-08-10T06:31:48,020 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691649107986/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-08-10T06:31:48,020 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/mapreduce.cluster.local.dir/1691649107986/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-08-10T06:31:48,072 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-08-10T06:31:48,119 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/25a1c5b32e608be67dca00df059e0053/f/06cc12c6a1d04d99aa771c07eb41201c_SeqId_4_ output=hdfs://localhost:45379/backupUT/backup_1691649094157/ns2/test-16916488687341/archive/data/ns2/test-16916488687341/25a1c5b32e608be67dca00df059e0053/f/06cc12c6a1d04d99aa771c07eb41201c_SeqId_4_ 2023-08-10T06:31:48,119 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8725 (8.5 K) time=0sec 8.321M/sec 2023-08-10T06:31:49,056 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-08-10T06:31:49,057 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-08-10T06:31:49,062 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1691649096473_ns2_test-16916488687341 2023-08-10T06:31:49,062 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1691649096473_ns2_test-16916488687341 finished. 2023-08-10T06:31:49,062 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(1014): ns3:test-16916488687342 2023-08-10T06:31:49,062 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(1014): ns2:test-16916488687341 2023-08-10T06:31:49,068 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691649094157 2023-08-10T06:31:49,068 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-08-10T06:31:49,473 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:45379/backupUT/backup_1691649094157/.backup.manifest 2023-08-10T06:31:49,473 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691649094157 2023-08-10T06:31:49,473 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-08-10T06:31:49,878 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:45379/backupUT/backup_1691649094157/.backup.manifest 2023-08-10T06:31:49,878 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(358): Backup backup_1691649094157 finished: type=FULL,tablelist=ns3:test-16916488687342;ns2:test-16916488687341,targetRootDir=hdfs://localhost:45379/backupUT,startts=1691649096291,completets=1691649109068,bytescopied=0 2023-08-10T06:31:49,878 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-08-10T06:31:49,878 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1691649100592_ns3_test-16916488687342 2023-08-10T06:31:49,878 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1691649100592_ns3_test-16916488687342" 2023-08-10T06:31:49,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1691649100592_ns3_test-16916488687342 2023-08-10T06:31:49,881 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1691649100592_ns3_test-16916488687342 for backup backup_1691649094157 succeeded. 2023-08-10T06:31:49,881 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1691649096473_ns2_test-16916488687341 2023-08-10T06:31:49,882 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1691649096473_ns2_test-16916488687341" 2023-08-10T06:31:49,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1691649096473_ns2_test-16916488687341 2023-08-10T06:31:49,884 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1691649096473_ns2_test-16916488687341 for backup backup_1691649094157 succeeded. 2023-08-10T06:31:49,884 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-08-10T06:31:49,888 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-08-10T06:31:49,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33705 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-08-10T06:31:49,890 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-08-10T06:31:49,891 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-08-10T06:31:49,896 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.TableBackupClient(376): Backup backup_1691649094157 completed. 2023-08-10T06:31:49,896 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:31:49,904 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(101): HBase table ns2:table2 does not exist. It will be created during restore process 2023-08-10T06:31:49,904 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(101): HBase table ns3:table3_restore does not exist. It will be created during restore process 2023-08-10T06:31:49,905 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:45379/backupUT/backup_1691649094157/.backup.manifest 2023-08-10T06:31:49,906 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649094157/.backup.manifest 2023-08-10T06:31:49,907 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:45379/backupUT/backup_1691649094157/.backup.manifest 2023-08-10T06:31:49,908 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649094157/.backup.manifest 2023-08-10T06:31:49,909 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:45379/backupUT/backup_1691649094157/.backup.manifest 2023-08-10T06:31:49,910 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649094157/.backup.manifest 2023-08-10T06:31:49,910 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] impl.RestoreTablesClient(148): Restoring 'faketable1' to 'ns2:table2' from full backup image hdfs://localhost:45379/backupUT/backup_1691649094157/default/faketable1 2023-08-10T06:31:49,927 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreMultipleDNE Thread=603 (was 604), OpenFileDescriptor=693 (was 684) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=77 (was 98), ProcessCount=169 (was 170), AvailableMemoryMB=1506 (was 1568) 2023-08-10T06:31:49,927 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.ResourceChecker(130): Thread=603 is superior to 500 2023-08-10T06:31:49,933 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2023-08-10T06:31:49,933 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:31:49,933 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2023-08-10T06:31:49,933 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] util.JVMClusterUtil(257): Found active master hash=550166155, stopped=false 2023-08-10T06:31:49,933 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] master.ServerManager(910): Cluster shutdown requested of master=jenkins-hbase3.apache.org,33705,1691648856595 2023-08-10T06:31:49,936 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] procedure2.ProcedureExecutor(631): Stopping 2023-08-10T06:31:49,937 DEBUG [Listener at jenkins-hbase3.apache.org/36923 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:31:49,937 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] regionserver.HRegionServer(2513): ***** STOPPING region server 'jenkins-hbase3.apache.org,45101,1691648859755' ***** 2023-08-10T06:31:49,937 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] regionserver.HRegionServer(2527): STOPPED: Shutdown requested 2023-08-10T06:31:49,937 INFO [RS:0;jenkins-hbase3:45101 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-08-10T06:31:49,939 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2023-08-10T06:31:49,942 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2023-08-10T06:31:49,951 INFO [RS:0;jenkins-hbase3:45101 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@71b422e2{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-08-10T06:31:49,956 INFO [RS:0;jenkins-hbase3:45101 {}] server.AbstractConnector(383): Stopped ServerConnector@39545045{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-08-10T06:31:49,956 INFO [RS:0;jenkins-hbase3:45101 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-08-10T06:31:49,957 INFO [RS:0;jenkins-hbase3:45101 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@6266f472{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-08-10T06:31:49,958 INFO [RS:0;jenkins-hbase3:45101 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@5d880175{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/hadoop.log.dir/,STOPPED} 2023-08-10T06:31:49,960 INFO [RS:0;jenkins-hbase3:45101 {}] regionserver.HeapMemoryManager(220): Stopping 2023-08-10T06:31:49,960 INFO [RS:0;jenkins-hbase3:45101 {}] flush.RegionServerFlushTableProcedureManager(117): Stopping region server flush procedure manager gracefully. 2023-08-10T06:31:49,960 INFO [RS:0;jenkins-hbase3:45101 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2023-08-10T06:31:49,961 INFO [RS:0;jenkins-hbase3:45101 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2023-08-10T06:31:49,961 INFO [RS:0;jenkins-hbase3:45101 {}] regionserver.HRegionServer(3524): Received CLOSE for 12a431f46365339f8fdb6b1e8b7a04c1 2023-08-10T06:31:49,961 INFO [RS:0;jenkins-hbase3:45101 {}] regionserver.HRegionServer(3524): Received CLOSE for b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:31:49,961 INFO [RS:0;jenkins-hbase3:45101 {}] regionserver.HRegionServer(3524): Received CLOSE for 285dade39f9f097a949721d4ab45d071 2023-08-10T06:31:49,961 INFO [RS:0;jenkins-hbase3:45101 {}] regionserver.HRegionServer(3524): Received CLOSE for 7ccd561208a3736e53c95b12322d6daf 2023-08-10T06:31:49,961 INFO [RS:0;jenkins-hbase3:45101 {}] regionserver.HRegionServer(3524): Received CLOSE for 513ad41c456db08f6287ccd38ca6b19c 2023-08-10T06:31:49,961 INFO [RS:0;jenkins-hbase3:45101 {}] regionserver.HRegionServer(3524): Received CLOSE for 25a1c5b32e608be67dca00df059e0053 2023-08-10T06:31:49,961 INFO [RS:0;jenkins-hbase3:45101 {}] regionserver.HRegionServer(3524): Received CLOSE for 547cc5444af6a8eb48a29d4d03cf49dd 2023-08-10T06:31:49,961 INFO [RS:0;jenkins-hbase3:45101 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,45101,1691648859755 2023-08-10T06:31:49,961 DEBUG [RS:0;jenkins-hbase3:45101 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:31:49,962 INFO [RS:0;jenkins-hbase3:45101 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2023-08-10T06:31:49,962 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 12a431f46365339f8fdb6b1e8b7a04c1, disabling compactions & flushes 2023-08-10T06:31:49,962 INFO [RS:0;jenkins-hbase3:45101 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2023-08-10T06:31:49,962 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1691648879270.12a431f46365339f8fdb6b1e8b7a04c1. 2023-08-10T06:31:49,962 INFO [RS:0;jenkins-hbase3:45101 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2023-08-10T06:31:49,962 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1691648879270.12a431f46365339f8fdb6b1e8b7a04c1. 2023-08-10T06:31:49,962 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1691648879270.12a431f46365339f8fdb6b1e8b7a04c1. after waiting 0 ms 2023-08-10T06:31:49,962 INFO [RS:0;jenkins-hbase3:45101 {}] regionserver.HRegionServer(3524): Received CLOSE for 1588230740 2023-08-10T06:31:49,962 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1691648879270.12a431f46365339f8fdb6b1e8b7a04c1. 2023-08-10T06:31:49,962 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 12a431f46365339f8fdb6b1e8b7a04c1 2/2 column families, dataSize=5.66 KB heapSize=8.50 KB 2023-08-10T06:31:49,963 INFO [RS:0;jenkins-hbase3:45101 {}] regionserver.HRegionServer(1579): Waiting on 8 regions to close 2023-08-10T06:31:49,963 DEBUG [RS:0;jenkins-hbase3:45101 {}] regionserver.HRegionServer(1583): Online Regions={12a431f46365339f8fdb6b1e8b7a04c1=backup:system_bulk,,1691648879270.12a431f46365339f8fdb6b1e8b7a04c1., b9c493f2c9fbef45de19e8268a534e72=backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72., 285dade39f9f097a949721d4ab45d071=ns4:test-16916488687343,,1691648875819.285dade39f9f097a949721d4ab45d071., 7ccd561208a3736e53c95b12322d6daf=hbase:namespace,,1691648863140.7ccd561208a3736e53c95b12322d6daf., 513ad41c456db08f6287ccd38ca6b19c=test-1691648868734,,1691648971834.513ad41c456db08f6287ccd38ca6b19c., 1588230740=hbase:meta,,1.1588230740, 25a1c5b32e608be67dca00df059e0053=ns2:test-16916488687341,,1691649034386.25a1c5b32e608be67dca00df059e0053., 547cc5444af6a8eb48a29d4d03cf49dd=ns3:test-16916488687342,,1691649039444.547cc5444af6a8eb48a29d4d03cf49dd.} 2023-08-10T06:31:49,963 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-08-10T06:31:49,963 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-08-10T06:31:49,963 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-08-10T06:31:49,964 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-08-10T06:31:49,964 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-08-10T06:31:49,966 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2819): Flushing 1588230740 3/3 column families, dataSize=38.00 KB heapSize=66.30 KB 2023-08-10T06:31:49,967 DEBUG [RS:0;jenkins-hbase3:45101 {}] regionserver.HRegionServer(1609): Waiting on 12a431f46365339f8fdb6b1e8b7a04c1, 1588230740, 25a1c5b32e608be67dca00df059e0053, 285dade39f9f097a949721d4ab45d071, 513ad41c456db08f6287ccd38ca6b19c, 547cc5444af6a8eb48a29d4d03cf49dd, 7ccd561208a3736e53c95b12322d6daf, b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:31:49,981 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system_bulk/12a431f46365339f8fdb6b1e8b7a04c1/.tmp/meta/a83774c9f2284b4ebcacfb02e49aa69c is 323, key is bulk:ns2:test-16916488687341:25a1c5b32e608be67dca00df059e0053:06cc12c6a1d04d99aa771c07eb41201c_SeqId_4_/meta:path/1691649039114/Put/seqid=0 2023-08-10T06:31:49,995 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/hbase/meta/1588230740/.tmp/info/042418442f384c498187b8242c8b5562 is 159, key is ns2:test-16916488687341,,1691649034386.25a1c5b32e608be67dca00df059e0053./info:regioninfo/1691649035542/Put/seqid=0 2023-08-10T06:31:50,026 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2023-08-10T06:31:50,026 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2023-08-10T06:31:50,167 DEBUG [RS:0;jenkins-hbase3:45101 {}] regionserver.HRegionServer(1609): Waiting on 12a431f46365339f8fdb6b1e8b7a04c1, 1588230740, 25a1c5b32e608be67dca00df059e0053, 285dade39f9f097a949721d4ab45d071, 513ad41c456db08f6287ccd38ca6b19c, 547cc5444af6a8eb48a29d4d03cf49dd, 7ccd561208a3736e53c95b12322d6daf, b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:31:50,367 DEBUG [RS:0;jenkins-hbase3:45101 {}] regionserver.HRegionServer(1609): Waiting on 12a431f46365339f8fdb6b1e8b7a04c1, 1588230740, 25a1c5b32e608be67dca00df059e0053, 285dade39f9f097a949721d4ab45d071, 513ad41c456db08f6287ccd38ca6b19c, 547cc5444af6a8eb48a29d4d03cf49dd, 7ccd561208a3736e53c95b12322d6daf, b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:31:50,386 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.66 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system_bulk/12a431f46365339f8fdb6b1e8b7a04c1/.tmp/meta/a83774c9f2284b4ebcacfb02e49aa69c 2023-08-10T06:31:50,391 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a83774c9f2284b4ebcacfb02e49aa69c 2023-08-10T06:31:50,392 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system_bulk/12a431f46365339f8fdb6b1e8b7a04c1/.tmp/meta/a83774c9f2284b4ebcacfb02e49aa69c as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system_bulk/12a431f46365339f8fdb6b1e8b7a04c1/meta/a83774c9f2284b4ebcacfb02e49aa69c 2023-08-10T06:31:50,396 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system_bulk/12a431f46365339f8fdb6b1e8b7a04c1/meta/a83774c9f2284b4ebcacfb02e49aa69c, entries=16, sequenceid=12, filesize=8.1 K 2023-08-10T06:31:50,397 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~5.66 KB/5800, heapSize ~8.23 KB/8432, currentSize=0 B/0 for 12a431f46365339f8fdb6b1e8b7a04c1 in 435ms, sequenceid=12, compaction requested=false 2023-08-10T06:31:50,400 DEBUG [StoreFileCloser-12a431f46365339f8fdb6b1e8b7a04c1-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a83774c9f2284b4ebcacfb02e49aa69c 2023-08-10T06:31:50,402 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.30 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/hbase/meta/1588230740/.tmp/info/042418442f384c498187b8242c8b5562 2023-08-10T06:31:50,415 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 042418442f384c498187b8242c8b5562 2023-08-10T06:31:50,424 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system_bulk/12a431f46365339f8fdb6b1e8b7a04c1/recovered.edits/15.seqid, newMaxSeqId=15, maxSeqId=1 2023-08-10T06:31:50,424 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:31:50,425 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system_bulk,,1691648879270.12a431f46365339f8fdb6b1e8b7a04c1. 2023-08-10T06:31:50,425 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 12a431f46365339f8fdb6b1e8b7a04c1: 2023-08-10T06:31:50,425 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system_bulk,,1691648879270.12a431f46365339f8fdb6b1e8b7a04c1. 2023-08-10T06:31:50,426 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing b9c493f2c9fbef45de19e8268a534e72, disabling compactions & flushes 2023-08-10T06:31:50,426 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. 2023-08-10T06:31:50,426 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. 2023-08-10T06:31:50,426 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. after waiting 0 ms 2023-08-10T06:31:50,426 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. 2023-08-10T06:31:50,426 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing b9c493f2c9fbef45de19e8268a534e72 2/2 column families, dataSize=1.46 KB heapSize=2.61 KB 2023-08-10T06:31:50,429 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/meta/a147542b499646b3ad6de60cdf84e355 is 171, key is trslm:hdfs://localhost:45379/backupUT\x00ns2:test-16916488687341/meta:log-roll-map/1691649109064/Put/seqid=0 2023-08-10T06:31:50,431 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/hbase/meta/1588230740/.tmp/rep_barrier/a5930b6ff62e4ee9ba57e0ef011c5abb is 107, key is ns2:test-16916488687341,,1691648871482.455351a9679f34a345fb40a29340fd37./rep_barrier:/1691648901069/DeleteFamily/seqid=0 2023-08-10T06:31:50,568 DEBUG [RS:0;jenkins-hbase3:45101 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 25a1c5b32e608be67dca00df059e0053, 285dade39f9f097a949721d4ab45d071, 513ad41c456db08f6287ccd38ca6b19c, 547cc5444af6a8eb48a29d4d03cf49dd, 7ccd561208a3736e53c95b12322d6daf, b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:31:50,768 DEBUG [RS:0;jenkins-hbase3:45101 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 25a1c5b32e608be67dca00df059e0053, 285dade39f9f097a949721d4ab45d071, 513ad41c456db08f6287ccd38ca6b19c, 547cc5444af6a8eb48a29d4d03cf49dd, 7ccd561208a3736e53c95b12322d6daf, b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:31:50,837 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=717 B at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/meta/a147542b499646b3ad6de60cdf84e355 2023-08-10T06:31:50,839 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.13 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/hbase/meta/1588230740/.tmp/rep_barrier/a5930b6ff62e4ee9ba57e0ef011c5abb 2023-08-10T06:31:50,842 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a147542b499646b3ad6de60cdf84e355 2023-08-10T06:31:50,844 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a5930b6ff62e4ee9ba57e0ef011c5abb 2023-08-10T06:31:50,845 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/session/70098b8ff24d48f5a3e8c6d5110fb118 is 524, key is session:backup_1691649094157/session:context/1691649109890/Put/seqid=0 2023-08-10T06:31:50,857 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/hbase/meta/1588230740/.tmp/table/e3468b2cce0645aa82062ebe1b133114 is 101, key is ns2:test-16916488687341,,1691648871482.455351a9679f34a345fb40a29340fd37./table:/1691648901069/DeleteFamily/seqid=0 2023-08-10T06:31:50,860 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.56 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/hbase/meta/1588230740/.tmp/table/e3468b2cce0645aa82062ebe1b133114 2023-08-10T06:31:50,864 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: e3468b2cce0645aa82062ebe1b133114 2023-08-10T06:31:50,865 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/hbase/meta/1588230740/.tmp/info/042418442f384c498187b8242c8b5562 as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/hbase/meta/1588230740/info/042418442f384c498187b8242c8b5562 2023-08-10T06:31:50,869 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/hbase/meta/1588230740/info/042418442f384c498187b8242c8b5562, entries=94, sequenceid=183, filesize=16.4 K 2023-08-10T06:31:50,870 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/hbase/meta/1588230740/.tmp/rep_barrier/a5930b6ff62e4ee9ba57e0ef011c5abb as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/hbase/meta/1588230740/rep_barrier/a5930b6ff62e4ee9ba57e0ef011c5abb 2023-08-10T06:31:50,874 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/hbase/meta/1588230740/rep_barrier/a5930b6ff62e4ee9ba57e0ef011c5abb, entries=12, sequenceid=183, filesize=6.5 K 2023-08-10T06:31:50,875 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/hbase/meta/1588230740/.tmp/table/e3468b2cce0645aa82062ebe1b133114 as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/hbase/meta/1588230740/table/e3468b2cce0645aa82062ebe1b133114 2023-08-10T06:31:50,880 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/hbase/meta/1588230740/table/e3468b2cce0645aa82062ebe1b133114, entries=32, sequenceid=183, filesize=7.3 K 2023-08-10T06:31:50,880 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3022): Finished flush of dataSize ~38.00 KB/38910, heapSize ~66.25 KB/67840, currentSize=0 B/0 for 1588230740 in 916ms, sequenceid=183, compaction requested=false 2023-08-10T06:31:50,883 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 042418442f384c498187b8242c8b5562 2023-08-10T06:31:50,885 DEBUG [StoreFileCloser-1588230740-rep_barrier-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a5930b6ff62e4ee9ba57e0ef011c5abb 2023-08-10T06:31:50,886 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: e3468b2cce0645aa82062ebe1b133114 2023-08-10T06:31:50,890 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/hbase/meta/1588230740/recovered.edits/186.seqid, newMaxSeqId=186, maxSeqId=1 2023-08-10T06:31:50,891 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:31:50,891 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-08-10T06:31:50,891 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-08-10T06:31:50,891 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-08-10T06:31:50,891 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2023-08-10T06:31:50,968 INFO [RS:0;jenkins-hbase3:45101 {}] regionserver.HRegionServer(1579): Waiting on 6 regions to close 2023-08-10T06:31:50,968 DEBUG [RS:0;jenkins-hbase3:45101 {}] regionserver.HRegionServer(1583): Online Regions={b9c493f2c9fbef45de19e8268a534e72=backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72., 285dade39f9f097a949721d4ab45d071=ns4:test-16916488687343,,1691648875819.285dade39f9f097a949721d4ab45d071., 7ccd561208a3736e53c95b12322d6daf=hbase:namespace,,1691648863140.7ccd561208a3736e53c95b12322d6daf., 513ad41c456db08f6287ccd38ca6b19c=test-1691648868734,,1691648971834.513ad41c456db08f6287ccd38ca6b19c., 25a1c5b32e608be67dca00df059e0053=ns2:test-16916488687341,,1691649034386.25a1c5b32e608be67dca00df059e0053., 547cc5444af6a8eb48a29d4d03cf49dd=ns3:test-16916488687342,,1691649039444.547cc5444af6a8eb48a29d4d03cf49dd.} 2023-08-10T06:31:50,968 DEBUG [RS:0;jenkins-hbase3:45101 {}] regionserver.HRegionServer(1609): Waiting on 25a1c5b32e608be67dca00df059e0053, 285dade39f9f097a949721d4ab45d071, 513ad41c456db08f6287ccd38ca6b19c, 547cc5444af6a8eb48a29d4d03cf49dd, 7ccd561208a3736e53c95b12322d6daf, b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:31:51,168 DEBUG [RS:0;jenkins-hbase3:45101 {}] regionserver.HRegionServer(1609): Waiting on 25a1c5b32e608be67dca00df059e0053, 285dade39f9f097a949721d4ab45d071, 513ad41c456db08f6287ccd38ca6b19c, 547cc5444af6a8eb48a29d4d03cf49dd, 7ccd561208a3736e53c95b12322d6daf, b9c493f2c9fbef45de19e8268a534e72 2023-08-10T06:31:51,251 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=775 B at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/session/70098b8ff24d48f5a3e8c6d5110fb118 2023-08-10T06:31:51,255 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 70098b8ff24d48f5a3e8c6d5110fb118 2023-08-10T06:31:51,256 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/meta/a147542b499646b3ad6de60cdf84e355 as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/a147542b499646b3ad6de60cdf84e355 2023-08-10T06:31:51,260 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/meta/a147542b499646b3ad6de60cdf84e355, entries=6, sequenceid=148, filesize=5.8 K 2023-08-10T06:31:51,261 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/.tmp/session/70098b8ff24d48f5a3e8c6d5110fb118 as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/70098b8ff24d48f5a3e8c6d5110fb118 2023-08-10T06:31:51,265 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/session/70098b8ff24d48f5a3e8c6d5110fb118, entries=2, sequenceid=148, filesize=5.5 K 2023-08-10T06:31:51,265 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~1.46 KB/1492, heapSize ~2.58 KB/2640, currentSize=0 B/0 for b9c493f2c9fbef45de19e8268a534e72 in 839ms, sequenceid=148, compaction requested=true 2023-08-10T06:31:51,267 DEBUG [StoreFileCloser-b9c493f2c9fbef45de19e8268a534e72-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: bc10682e7f36491d941e2fe62e2e4100 2023-08-10T06:31:51,269 DEBUG [StoreFileCloser-b9c493f2c9fbef45de19e8268a534e72-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 9d9a4feb1c4e44d99cc63e3f68878731 2023-08-10T06:31:51,270 DEBUG [StoreFileCloser-b9c493f2c9fbef45de19e8268a534e72-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: f04a99990b2c429eaf1096ef413be3b8 2023-08-10T06:31:51,272 DEBUG [StoreFileCloser-b9c493f2c9fbef45de19e8268a534e72-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 9e97dc38b66b476f952990b23fa0a3ca 2023-08-10T06:31:51,273 DEBUG [StoreFileCloser-b9c493f2c9fbef45de19e8268a534e72-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 930c578fe3f543e1b6308abfb31c7246 2023-08-10T06:31:51,275 DEBUG [StoreFileCloser-b9c493f2c9fbef45de19e8268a534e72-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 24713cefebe0499c812c6d209dea0fd6 2023-08-10T06:31:51,276 DEBUG [StoreFileCloser-b9c493f2c9fbef45de19e8268a534e72-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 0ae449898702485faa63081b0f3f028a 2023-08-10T06:31:51,279 DEBUG [StoreFileCloser-b9c493f2c9fbef45de19e8268a534e72-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: cec52c43e8824ebfbbc2cf17b7a9ec5f 2023-08-10T06:31:51,283 DEBUG [StoreFileCloser-b9c493f2c9fbef45de19e8268a534e72-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 8f5b1bc0de87458e96bd38212599dcbd 2023-08-10T06:31:51,283 DEBUG [StoreFileCloser-b9c493f2c9fbef45de19e8268a534e72-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 184a27fc671b4e9085d56df41f1234f5 2023-08-10T06:31:51,283 DEBUG [StoreFileCloser-b9c493f2c9fbef45de19e8268a534e72-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 225d36cdae6a48eb86402cbd55c85ab2 2023-08-10T06:31:51,283 DEBUG [StoreFileCloser-b9c493f2c9fbef45de19e8268a534e72-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 07b4147d42b3429baac41e09e439933a 2023-08-10T06:31:51,283 DEBUG [StoreFileCloser-b9c493f2c9fbef45de19e8268a534e72-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a147542b499646b3ad6de60cdf84e355 2023-08-10T06:31:51,283 DEBUG [StoreFileCloser-b9c493f2c9fbef45de19e8268a534e72-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: e88678d5b4b14eecabac817daf12539e 2023-08-10T06:31:51,283 DEBUG [StoreFileCloser-b9c493f2c9fbef45de19e8268a534e72-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 37561b9a8ae2406bb6d053c33aeff31c 2023-08-10T06:31:51,284 DEBUG [StoreFileCloser-b9c493f2c9fbef45de19e8268a534e72-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 883911c19cab473f8bf801f934c90bc1 2023-08-10T06:31:51,284 DEBUG [StoreFileCloser-b9c493f2c9fbef45de19e8268a534e72-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 0c7705a45c554d6a97fa37fac319ceff 2023-08-10T06:31:51,284 DEBUG [StoreFileCloser-b9c493f2c9fbef45de19e8268a534e72-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 398a15bfffb84acfadb1be252c2e133a 2023-08-10T06:31:51,284 DEBUG [StoreFileCloser-b9c493f2c9fbef45de19e8268a534e72-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: adceb6320b02451abd02beed6c69c000 2023-08-10T06:31:51,284 DEBUG [StoreFileCloser-b9c493f2c9fbef45de19e8268a534e72-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 68fd0cf6c3094cc7af1a5176829fd9a4 2023-08-10T06:31:51,284 DEBUG [StoreFileCloser-b9c493f2c9fbef45de19e8268a534e72-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 40f9fbdeac5b49dbb654e429c177708d 2023-08-10T06:31:51,284 DEBUG [StoreFileCloser-b9c493f2c9fbef45de19e8268a534e72-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 75226fb0202e42819224b5e5920a02f2 2023-08-10T06:31:51,284 DEBUG [StoreFileCloser-b9c493f2c9fbef45de19e8268a534e72-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 3702fe0322c94bcc85372b5eb9c99029 2023-08-10T06:31:51,285 DEBUG [StoreFileCloser-b9c493f2c9fbef45de19e8268a534e72-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 8a873f56ce4e4af992eb65b5390f76c2 2023-08-10T06:31:51,285 DEBUG [StoreFileCloser-b9c493f2c9fbef45de19e8268a534e72-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 1dd3913d29eb48739f0c6f2120db1ded 2023-08-10T06:31:51,285 DEBUG [StoreFileCloser-b9c493f2c9fbef45de19e8268a534e72-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: ea1ed59991114105a8ce717031f62056 2023-08-10T06:31:51,285 DEBUG [StoreFileCloser-b9c493f2c9fbef45de19e8268a534e72-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 70098b8ff24d48f5a3e8c6d5110fb118 2023-08-10T06:31:51,288 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/backup/system/b9c493f2c9fbef45de19e8268a534e72/recovered.edits/151.seqid, newMaxSeqId=151, maxSeqId=1 2023-08-10T06:31:51,288 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:31:51,289 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. 2023-08-10T06:31:51,289 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for b9c493f2c9fbef45de19e8268a534e72: 2023-08-10T06:31:51,289 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system,,1691648878142.b9c493f2c9fbef45de19e8268a534e72. 2023-08-10T06:31:51,290 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 285dade39f9f097a949721d4ab45d071, disabling compactions & flushes 2023-08-10T06:31:51,290 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns4:test-16916488687343,,1691648875819.285dade39f9f097a949721d4ab45d071. 2023-08-10T06:31:51,290 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16916488687343,,1691648875819.285dade39f9f097a949721d4ab45d071. 2023-08-10T06:31:51,290 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16916488687343,,1691648875819.285dade39f9f097a949721d4ab45d071. after waiting 0 ms 2023-08-10T06:31:51,290 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16916488687343,,1691648875819.285dade39f9f097a949721d4ab45d071. 2023-08-10T06:31:51,293 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns4/test-16916488687343/285dade39f9f097a949721d4ab45d071/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-08-10T06:31:51,293 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:31:51,294 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns4:test-16916488687343,,1691648875819.285dade39f9f097a949721d4ab45d071. 2023-08-10T06:31:51,294 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 285dade39f9f097a949721d4ab45d071: 2023-08-10T06:31:51,294 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns4:test-16916488687343,,1691648875819.285dade39f9f097a949721d4ab45d071. 2023-08-10T06:31:51,295 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 7ccd561208a3736e53c95b12322d6daf, disabling compactions & flushes 2023-08-10T06:31:51,295 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:namespace,,1691648863140.7ccd561208a3736e53c95b12322d6daf. 2023-08-10T06:31:51,295 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1691648863140.7ccd561208a3736e53c95b12322d6daf. 2023-08-10T06:31:51,295 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1691648863140.7ccd561208a3736e53c95b12322d6daf. after waiting 0 ms 2023-08-10T06:31:51,295 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1691648863140.7ccd561208a3736e53c95b12322d6daf. 2023-08-10T06:31:51,296 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 7ccd561208a3736e53c95b12322d6daf 1/1 column families, dataSize=249 B heapSize=1.02 KB 2023-08-10T06:31:51,308 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/hbase/namespace/7ccd561208a3736e53c95b12322d6daf/.tmp/info/5f6d4ceea59c4066b4ed8d7665fa390b is 45, key is default/info:d/1691648864116/Put/seqid=0 2023-08-10T06:31:51,369 DEBUG [RS:0;jenkins-hbase3:45101 {}] regionserver.HRegionServer(1609): Waiting on 25a1c5b32e608be67dca00df059e0053, 513ad41c456db08f6287ccd38ca6b19c, 547cc5444af6a8eb48a29d4d03cf49dd, 7ccd561208a3736e53c95b12322d6daf 2023-08-10T06:31:51,569 DEBUG [RS:0;jenkins-hbase3:45101 {}] regionserver.HRegionServer(1609): Waiting on 25a1c5b32e608be67dca00df059e0053, 513ad41c456db08f6287ccd38ca6b19c, 547cc5444af6a8eb48a29d4d03cf49dd, 7ccd561208a3736e53c95b12322d6daf 2023-08-10T06:31:51,641 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(140): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=jenkins-hbase3.apache.org:33705 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:217) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:450) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:335) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:93) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:599) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.getLiveRegionServers(RegionServerStatusProtos.java:18041) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionImplementation.getLiveRegionServers(ConnectionImplementation.java:2363) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:138) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_362] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) ~[?:1.8.0_362] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:365) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:447) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] ... 13 more 2023-08-10T06:31:51,712 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/hbase/namespace/7ccd561208a3736e53c95b12322d6daf/.tmp/info/5f6d4ceea59c4066b4ed8d7665fa390b 2023-08-10T06:31:51,717 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 5f6d4ceea59c4066b4ed8d7665fa390b 2023-08-10T06:31:51,717 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/hbase/namespace/7ccd561208a3736e53c95b12322d6daf/.tmp/info/5f6d4ceea59c4066b4ed8d7665fa390b as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/hbase/namespace/7ccd561208a3736e53c95b12322d6daf/info/5f6d4ceea59c4066b4ed8d7665fa390b 2023-08-10T06:31:51,723 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/hbase/namespace/7ccd561208a3736e53c95b12322d6daf/info/5f6d4ceea59c4066b4ed8d7665fa390b, entries=7, sequenceid=11, filesize=5.1 K 2023-08-10T06:31:51,723 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~249 B/249, heapSize ~1.01 KB/1032, currentSize=0 B/0 for 7ccd561208a3736e53c95b12322d6daf in 428ms, sequenceid=11, compaction requested=false 2023-08-10T06:31:51,726 DEBUG [StoreFileCloser-7ccd561208a3736e53c95b12322d6daf-info-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 5f6d4ceea59c4066b4ed8d7665fa390b 2023-08-10T06:31:51,730 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/hbase/namespace/7ccd561208a3736e53c95b12322d6daf/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2023-08-10T06:31:51,730 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:31:51,731 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:namespace,,1691648863140.7ccd561208a3736e53c95b12322d6daf. 2023-08-10T06:31:51,731 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 7ccd561208a3736e53c95b12322d6daf: 2023-08-10T06:31:51,731 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1691648863140.7ccd561208a3736e53c95b12322d6daf. 2023-08-10T06:31:51,731 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 513ad41c456db08f6287ccd38ca6b19c, disabling compactions & flushes 2023-08-10T06:31:51,731 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region test-1691648868734,,1691648971834.513ad41c456db08f6287ccd38ca6b19c. 2023-08-10T06:31:51,731 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1691648868734,,1691648971834.513ad41c456db08f6287ccd38ca6b19c. 2023-08-10T06:31:51,731 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on test-1691648868734,,1691648971834.513ad41c456db08f6287ccd38ca6b19c. after waiting 0 ms 2023-08-10T06:31:51,731 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region test-1691648868734,,1691648971834.513ad41c456db08f6287ccd38ca6b19c. 2023-08-10T06:31:51,731 DEBUG [StoreFileCloser-513ad41c456db08f6287ccd38ca6b19c-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 8eec1fbc8346444d9f3bf9d53b64b2b2_SeqId_4_ 2023-08-10T06:31:51,734 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/default/test-1691648868734/513ad41c456db08f6287ccd38ca6b19c/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2023-08-10T06:31:51,734 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:31:51,735 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed test-1691648868734,,1691648971834.513ad41c456db08f6287ccd38ca6b19c. 2023-08-10T06:31:51,735 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 513ad41c456db08f6287ccd38ca6b19c: 2023-08-10T06:31:51,735 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test-1691648868734,,1691648971834.513ad41c456db08f6287ccd38ca6b19c. 2023-08-10T06:31:51,735 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 25a1c5b32e608be67dca00df059e0053, disabling compactions & flushes 2023-08-10T06:31:51,735 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns2:test-16916488687341,,1691649034386.25a1c5b32e608be67dca00df059e0053. 2023-08-10T06:31:51,735 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16916488687341,,1691649034386.25a1c5b32e608be67dca00df059e0053. 2023-08-10T06:31:51,735 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16916488687341,,1691649034386.25a1c5b32e608be67dca00df059e0053. after waiting 0 ms 2023-08-10T06:31:51,735 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16916488687341,,1691649034386.25a1c5b32e608be67dca00df059e0053. 2023-08-10T06:31:51,735 DEBUG [StoreFileCloser-25a1c5b32e608be67dca00df059e0053-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 06cc12c6a1d04d99aa771c07eb41201c_SeqId_4_ 2023-08-10T06:31:51,738 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns2/test-16916488687341/25a1c5b32e608be67dca00df059e0053/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2023-08-10T06:31:51,738 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:31:51,739 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns2:test-16916488687341,,1691649034386.25a1c5b32e608be67dca00df059e0053. 2023-08-10T06:31:51,739 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 25a1c5b32e608be67dca00df059e0053: 2023-08-10T06:31:51,739 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:test-16916488687341,,1691649034386.25a1c5b32e608be67dca00df059e0053. 2023-08-10T06:31:51,739 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 547cc5444af6a8eb48a29d4d03cf49dd, disabling compactions & flushes 2023-08-10T06:31:51,739 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns3:test-16916488687342,,1691649039444.547cc5444af6a8eb48a29d4d03cf49dd. 2023-08-10T06:31:51,739 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16916488687342,,1691649039444.547cc5444af6a8eb48a29d4d03cf49dd. 2023-08-10T06:31:51,739 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16916488687342,,1691649039444.547cc5444af6a8eb48a29d4d03cf49dd. after waiting 0 ms 2023-08-10T06:31:51,739 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16916488687342,,1691649039444.547cc5444af6a8eb48a29d4d03cf49dd. 2023-08-10T06:31:51,741 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/data/ns3/test-16916488687342/547cc5444af6a8eb48a29d4d03cf49dd/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2023-08-10T06:31:51,741 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:31:51,742 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns3:test-16916488687342,,1691649039444.547cc5444af6a8eb48a29d4d03cf49dd. 2023-08-10T06:31:51,742 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 547cc5444af6a8eb48a29d4d03cf49dd: 2023-08-10T06:31:51,742 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns3:test-16916488687342,,1691649039444.547cc5444af6a8eb48a29d4d03cf49dd. 2023-08-10T06:31:51,769 INFO [RS:0;jenkins-hbase3:45101 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,45101,1691648859755; all regions closed. 2023-08-10T06:31:51,779 DEBUG [RS:0;jenkins-hbase3:45101 {}] wal.AbstractFSWAL(1062): Moved 10 WAL file(s) to /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/oldWALs 2023-08-10T06:31:51,779 INFO [RS:0;jenkins-hbase3:45101 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C45101%2C1691648859755.meta:.meta(num 1691649096309) 2023-08-10T06:31:51,787 DEBUG [RS:0;jenkins-hbase3:45101 {}] wal.AbstractFSWAL(1062): Moved 7 WAL file(s) to /user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/oldWALs 2023-08-10T06:31:51,787 INFO [RS:0;jenkins-hbase3:45101 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C45101%2C1691648859755:(num 1691649096325) 2023-08-10T06:31:51,787 DEBUG [RS:0;jenkins-hbase3:45101 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:31:51,787 INFO [RS:0;jenkins-hbase3:45101 {}] regionserver.LeaseManager(133): Closed leases 2023-08-10T06:31:51,787 INFO [RS:0;jenkins-hbase3:45101 {}] hbase.ChoreService(370): Chore service for: regionserver/jenkins-hbase3:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown 2023-08-10T06:31:51,787 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-08-10T06:31:51,788 INFO [RS:0;jenkins-hbase3:45101 {}] ipc.NettyRpcServer(213): Stopping server on /172.31.12.81:45101 2023-08-10T06:31:51,799 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [jenkins-hbase3.apache.org,45101,1691648859755] 2023-08-10T06:31:51,800 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing jenkins-hbase3.apache.org,45101,1691648859755; numProcessing=1 2023-08-10T06:31:51,801 INFO [RegionServerTracker-0 {}] master.ServerManager(569): Cluster shutdown set; jenkins-hbase3.apache.org,45101,1691648859755 expired; onlineServers=0 2023-08-10T06:31:51,801 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2513): ***** STOPPING region server 'jenkins-hbase3.apache.org,33705,1691648856595' ***** 2023-08-10T06:31:51,801 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2527): STOPPED: Cluster shutdown set; onlineServer=0 2023-08-10T06:31:51,802 DEBUG [M:0;jenkins-hbase3:33705 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@473330bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-08-10T06:31:51,802 INFO [M:0;jenkins-hbase3:33705 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-08-10T06:31:51,848 INFO [M:0;jenkins-hbase3:33705 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@3eb692b5{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-08-10T06:31:51,848 INFO [M:0;jenkins-hbase3:33705 {}] server.AbstractConnector(383): Stopped ServerConnector@1949aa43{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-08-10T06:31:51,848 INFO [M:0;jenkins-hbase3:33705 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-08-10T06:31:51,849 INFO [M:0;jenkins-hbase3:33705 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@3d08223b{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-08-10T06:31:51,850 INFO [M:0;jenkins-hbase3:33705 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@71341aff{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/hadoop.log.dir/,STOPPED} 2023-08-10T06:31:51,850 INFO [M:0;jenkins-hbase3:33705 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,33705,1691648856595 2023-08-10T06:31:51,850 INFO [M:0;jenkins-hbase3:33705 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,33705,1691648856595; all regions closed. 2023-08-10T06:31:51,850 DEBUG [M:0;jenkins-hbase3:33705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:31:51,850 INFO [M:0;jenkins-hbase3:33705 {}] master.HMaster(1664): Stopping master jetty server 2023-08-10T06:31:51,851 INFO [M:0;jenkins-hbase3:33705 {}] server.AbstractConnector(383): Stopped ServerConnector@644017ac{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-08-10T06:31:51,852 DEBUG [M:0;jenkins-hbase3:33705 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2023-08-10T06:31:51,852 INFO [M:0;jenkins-hbase3:33705 {}] master.BackupLogCleaner(171): Stopping BackupLogCleaner 2023-08-10T06:31:51,852 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2023-08-10T06:31:51,852 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1691648861846 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1691648861846,5,FailOnTimeoutGroup] 2023-08-10T06:31:51,852 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1691648861845 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1691648861845,5,FailOnTimeoutGroup] 2023-08-10T06:31:51,852 DEBUG [M:0;jenkins-hbase3:33705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:31:51,853 DEBUG [M:0;jenkins-hbase3:33705 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2023-08-10T06:31:51,853 INFO [M:0;jenkins-hbase3:33705 {}] hbase.ChoreService(370): Chore service for: master/jenkins-hbase3:0 had [] on shutdown 2023-08-10T06:31:51,853 DEBUG [M:0;jenkins-hbase3:33705 {}] master.HMaster(1687): Stopping service threads 2023-08-10T06:31:51,853 INFO [M:0;jenkins-hbase3:33705 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2023-08-10T06:31:51,854 ERROR [M:0;jenkins-hbase3:33705 {}] procedure2.ProcedureExecutor(655): ThreadGroup java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] contains running threads; null: See STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-12,5,PEWorkerGroup] 2023-08-10T06:31:51,854 INFO [M:0;jenkins-hbase3:33705 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2023-08-10T06:31:51,854 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2023-08-10T06:31:51,855 WARN [M:0;jenkins-hbase3:33705 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2023-08-10T06:31:51,855 INFO [M:0;jenkins-hbase3:33705 {}] assignment.AssignmentManager(382): Stopping assignment manager 2023-08-10T06:31:51,855 INFO [M:0;jenkins-hbase3:33705 {}] region.MasterRegion(178): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2023-08-10T06:31:51,855 DEBUG [M:0;jenkins-hbase3:33705 {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-08-10T06:31:51,873 INFO [M:0;jenkins-hbase3:33705 {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-08-10T06:31:51,873 DEBUG [M:0;jenkins-hbase3:33705 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-08-10T06:31:51,873 DEBUG [M:0;jenkins-hbase3:33705 {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-08-10T06:31:51,873 DEBUG [M:0;jenkins-hbase3:33705 {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-08-10T06:31:51,873 INFO [M:0;jenkins-hbase3:33705 {}] regionserver.HRegion(2819): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=621.82 KB heapSize=756.55 KB 2023-08-10T06:31:51,884 DEBUG [M:0;jenkins-hbase3:33705 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/674376d09dbe42b0a68c65d9ba40c03d is 88, key is hbase:meta,,1/info:sn/1691648862459/Put/seqid=0 2023-08-10T06:31:51,888 INFO [M:0;jenkins-hbase3:33705 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=530 B at sequenceid=1616 (bloomFilter=true), to=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/674376d09dbe42b0a68c65d9ba40c03d 2023-08-10T06:31:51,893 DEBUG [M:0;jenkins-hbase3:33705 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 674376d09dbe42b0a68c65d9ba40c03d 2023-08-10T06:31:51,899 INFO [RS:0;jenkins-hbase3:45101 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,45101,1691648859755; zookeeper connection closed. 2023-08-10T06:31:51,899 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1755dbaf {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1755dbaf 2023-08-10T06:31:51,899 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2023-08-10T06:31:51,907 DEBUG [M:0;jenkins-hbase3:33705 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/69a22cfdfd7f46038c0a084af3ab8e0e is 987, key is \x00\x00\x00\x00\x00\x00\x00\x1D/proc:d/1691648880461/Put/seqid=0 2023-08-10T06:31:51,910 INFO [M:0;jenkins-hbase3:33705 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=621.22 KB at sequenceid=1616 (bloomFilter=true), to=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/69a22cfdfd7f46038c0a084af3ab8e0e 2023-08-10T06:31:51,914 INFO [M:0;jenkins-hbase3:33705 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 69a22cfdfd7f46038c0a084af3ab8e0e 2023-08-10T06:31:51,914 DEBUG [M:0;jenkins-hbase3:33705 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 69a22cfdfd7f46038c0a084af3ab8e0e 2023-08-10T06:31:51,930 DEBUG [M:0;jenkins-hbase3:33705 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7f9712f115ab4e0883560d83bdf921a9 is 82, key is jenkins-hbase3.apache.org,45101,1691648859755/rs:state/1691648861874/Put/seqid=0 2023-08-10T06:31:52,334 INFO [M:0;jenkins-hbase3:33705 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=1616 (bloomFilter=true), to=hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7f9712f115ab4e0883560d83bdf921a9 2023-08-10T06:31:52,338 DEBUG [M:0;jenkins-hbase3:33705 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 7f9712f115ab4e0883560d83bdf921a9 2023-08-10T06:31:52,339 DEBUG [M:0;jenkins-hbase3:33705 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/674376d09dbe42b0a68c65d9ba40c03d as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/674376d09dbe42b0a68c65d9ba40c03d 2023-08-10T06:31:52,343 INFO [M:0;jenkins-hbase3:33705 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/674376d09dbe42b0a68c65d9ba40c03d, entries=8, sequenceid=1616, filesize=5.5 K 2023-08-10T06:31:52,344 DEBUG [M:0;jenkins-hbase3:33705 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/69a22cfdfd7f46038c0a084af3ab8e0e as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/69a22cfdfd7f46038c0a084af3ab8e0e 2023-08-10T06:31:52,347 INFO [M:0;jenkins-hbase3:33705 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 69a22cfdfd7f46038c0a084af3ab8e0e 2023-08-10T06:31:52,347 INFO [M:0;jenkins-hbase3:33705 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/69a22cfdfd7f46038c0a084af3ab8e0e, entries=173, sequenceid=1616, filesize=39.1 K 2023-08-10T06:31:52,348 DEBUG [M:0;jenkins-hbase3:33705 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7f9712f115ab4e0883560d83bdf921a9 as hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7f9712f115ab4e0883560d83bdf921a9 2023-08-10T06:31:52,352 INFO [M:0;jenkins-hbase3:33705 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45379/user/jenkins/test-data/295a6edf-b1d4-b612-3a8c-f23952be085e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7f9712f115ab4e0883560d83bdf921a9, entries=1, sequenceid=1616, filesize=5.1 K 2023-08-10T06:31:52,353 INFO [M:0;jenkins-hbase3:33705 {}] regionserver.HRegion(3022): Finished flush of dataSize ~621.82 KB/636739, heapSize ~756.25 KB/774400, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 480ms, sequenceid=1616, compaction requested=false 2023-08-10T06:31:52,355 DEBUG [StoreFileCloser-1595e783b53d99cd5eef43b6debb2682-info-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 674376d09dbe42b0a68c65d9ba40c03d 2023-08-10T06:31:52,356 DEBUG [StoreFileCloser-1595e783b53d99cd5eef43b6debb2682-proc-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 69a22cfdfd7f46038c0a084af3ab8e0e 2023-08-10T06:31:52,358 DEBUG [StoreFileCloser-1595e783b53d99cd5eef43b6debb2682-rs-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 7f9712f115ab4e0883560d83bdf921a9 2023-08-10T06:31:52,358 INFO [M:0;jenkins-hbase3:33705 {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-08-10T06:31:52,358 DEBUG [M:0;jenkins-hbase3:33705 {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-08-10T06:31:52,361 INFO [M:0;jenkins-hbase3:33705 {}] flush.MasterFlushTableProcedureManager(81): stop: server shutting down. 2023-08-10T06:31:52,361 INFO [M:0;jenkins-hbase3:33705 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2023-08-10T06:31:52,362 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-08-10T06:31:52,362 INFO [M:0;jenkins-hbase3:33705 {}] ipc.NettyRpcServer(213): Stopping server on /172.31.12.81:33705 2023-08-10T06:31:52,467 INFO [M:0;jenkins-hbase3:33705 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,33705,1691648856595; zookeeper connection closed. 2023-08-10T06:31:52,475 WARN [Listener at jenkins-hbase3.apache.org/36923 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-08-10T06:31:52,484 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-08-10T06:31:52,593 WARN [BP-1233959968-172.31.12.81-1691648853138 heartbeating to localhost/127.0.0.1:45379 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-08-10T06:31:52,593 WARN [BP-1233959968-172.31.12.81-1691648853138 heartbeating to localhost/127.0.0.1:45379 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-1233959968-172.31.12.81-1691648853138 (Datanode Uuid 7c55e065-91e0-4898-9c2b-da8e30448879) service to localhost/127.0.0.1:45379 2023-08-10T06:31:52,596 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/cluster_28117de4-e82e-18fe-7324-f4a37a8c1f1e/dfs/data/data1/current/BP-1233959968-172.31.12.81-1691648853138 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-08-10T06:31:52,596 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeffb9de-a5cf-a638-68f8-d1ea731c2ddc/cluster_28117de4-e82e-18fe-7324-f4a37a8c1f1e/dfs/data/data2/current/BP-1233959968-172.31.12.81-1691648853138 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-08-10T06:31:52,636 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-08-10T06:31:52,861 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2023-08-10T06:31:52,861 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.HBaseTestingUtility(2857): Stopping mini mapreduce cluster... 2023-08-10T06:31:52,872 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-08-10T06:31:52,988 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-08-10T06:31:53,099 ERROR [Thread[Thread-275,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-08-10T06:31:53,100 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-08-10T06:31:53,207 WARN [ApplicationMaster Launcher {}] amlauncher.ApplicationMasterLauncher$LauncherThread(122): org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher$LauncherThread interrupted. Returning. 2023-08-10T06:31:53,214 ERROR [SchedulerEventDispatcher:Event Processor {}] event.EventDispatcher$EventProcessor(61): Returning, interrupted : java.lang.InterruptedException 2023-08-10T06:31:53,215 ERROR [Thread[Thread-285,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-08-10T06:31:53,220 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-08-10T06:31:53,322 ERROR [Thread[Thread-250,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-08-10T06:31:53,322 INFO [Listener at jenkins-hbase3.apache.org/36923 {}] hbase.HBaseTestingUtility(2860): Mini mapreduce cluster stopped